{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/bhushansonawaneint/miniconda3/envs/py35/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.onnx\n",
    "from onnx_coreml import convert\n",
    "from pytorch_transformers import *\n",
    "import numpy as np\n",
    "from utils import _compute_SNR"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Converting PyTorch Model into CoreML model\n",
    "1. Convert PyTorch to ONNX using PyTorch ONNX Export\n",
    "2. Convert ONNX model from step 1 into CoreML model using onnx-coreml converter\n",
    "\n",
    "Following notebook will go through converting Huggingface's BERT model into CoreML model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Description\n",
    "* https://huggingface.co/transformers/pretrained_models.html"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1 BERT BASE UNCASED\n",
    "- Please find saved PyTorch, ONNX and CoreML Model https://drive.google.com/drive/u/3/folders/1V4BxddAZ_EzQk18PRSu4lGjAQGqRfxU3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model path\n",
    "TMP_DIR = '/tmp/'\n",
    "model_name = 'bert-base-uncased'\n",
    "pt_path = TMP_DIR + model_name + '.pt'\n",
    "onnx_model_path = TMP_DIR + model_name + '.onnx'\n",
    "mlmodel_path  = TMP_DIR + model_name + '.mlmodel'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load BERT Base Model\n",
    "# Details: 12-layer, 768-hidden, 12-heads, 110M parameters. Trained on cased English text.\n",
    "model = BertModel.from_pretrained(model_name)\n",
    "torch.save(model, pt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Step 1 - Convert from PyTorch to ONNX\n",
    "test_input = torch.randint(0, 512, (1, 512))\n",
    "torch.onnx.export(model,\n",
    "                  test_input,\n",
    "                  onnx_model_path,\n",
    "                  input_names=[\"input_ids\"],\n",
    "                  output_names=[\"start_scores\", \"end_scores\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/960: Converting Node Type ConstantOfShape\n",
      "2/960: Converting Node Type ConstantOfShape\n",
      "3/960: Converting Node Type Unsqueeze\n",
      "4/960: Converting Node Type Unsqueeze\n",
      "5/960: Converting Node Type Sub\n",
      "6/960: Converting Node Type Mul\n",
      "7/960: Converting Node Type Expand\n",
      "8/960: Converting Node Type Gather\n",
      "9/960: Converting Node Type Gather\n",
      "10/960: Converting Node Type Gather\n",
      "11/960: Converting Node Type Add\n",
      "12/960: Converting Node Type Add\n",
      "13/960: Converting Node Type ReduceMean\n",
      "14/960: Converting Node Type Sub\n",
      "15/960: Converting Node Type Pow\n",
      "16/960: Converting Node Type ReduceMean\n",
      "17/960: Converting Node Type Add\n",
      "18/960: Converting Node Type Sqrt\n",
      "19/960: Converting Node Type Div\n",
      "20/960: Converting Node Type Mul\n",
      "21/960: Converting Node Type Add\n",
      "22/960: Converting Node Type MatMul\n",
      "23/960: Converting Node Type Add\n",
      "24/960: Converting Node Type MatMul\n",
      "25/960: Converting Node Type Add\n",
      "26/960: Converting Node Type MatMul\n",
      "27/960: Converting Node Type Add\n",
      "28/960: Converting Node Type Shape\n",
      "29/960: Converting Node Type Gather\n",
      "30/960: Converting Node Type Shape\n",
      "31/960: Converting Node Type Gather\n",
      "32/960: Converting Node Type Unsqueeze\n",
      "33/960: Converting Node Type Unsqueeze\n",
      "34/960: Converting Node Type Concat\n",
      "35/960: Converting Node Type Reshape\n",
      "36/960: Converting Node Type Transpose\n",
      "37/960: Converting Node Type Shape\n",
      "38/960: Converting Node Type Gather\n",
      "39/960: Converting Node Type Shape\n",
      "40/960: Converting Node Type Gather\n",
      "41/960: Converting Node Type Unsqueeze\n",
      "42/960: Converting Node Type Unsqueeze\n",
      "43/960: Converting Node Type Concat\n",
      "44/960: Converting Node Type Reshape\n",
      "45/960: Converting Node Type Shape\n",
      "46/960: Converting Node Type Gather\n",
      "47/960: Converting Node Type Shape\n",
      "48/960: Converting Node Type Gather\n",
      "49/960: Converting Node Type Unsqueeze\n",
      "50/960: Converting Node Type Unsqueeze\n",
      "51/960: Converting Node Type Concat\n",
      "52/960: Converting Node Type Reshape\n",
      "53/960: Converting Node Type Transpose\n",
      "54/960: Converting Node Type Transpose\n",
      "55/960: Converting Node Type MatMul\n",
      "56/960: Converting Node Type Div\n",
      "57/960: Converting Node Type Add\n",
      "58/960: Converting Node Type Softmax\n",
      "59/960: Converting Node Type MatMul\n",
      "60/960: Converting Node Type Transpose\n",
      "61/960: Converting Node Type Shape\n",
      "62/960: Converting Node Type Gather\n",
      "63/960: Converting Node Type Shape\n",
      "64/960: Converting Node Type Gather\n",
      "65/960: Converting Node Type Unsqueeze\n",
      "66/960: Converting Node Type Unsqueeze\n",
      "67/960: Converting Node Type Concat\n",
      "68/960: Converting Node Type Reshape\n",
      "69/960: Converting Node Type MatMul\n",
      "70/960: Converting Node Type Add\n",
      "71/960: Converting Node Type Add\n",
      "72/960: Converting Node Type ReduceMean\n",
      "73/960: Converting Node Type Sub\n",
      "74/960: Converting Node Type Pow\n",
      "75/960: Converting Node Type ReduceMean\n",
      "76/960: Converting Node Type Add\n",
      "77/960: Converting Node Type Sqrt\n",
      "78/960: Converting Node Type Div\n",
      "79/960: Converting Node Type Mul\n",
      "80/960: Converting Node Type Add\n",
      "81/960: Converting Node Type MatMul\n",
      "82/960: Converting Node Type Add\n",
      "83/960: Converting Node Type Mul\n",
      "84/960: Converting Node Type Div\n",
      "85/960: Converting Node Type Erf\n",
      "86/960: Converting Node Type Add\n",
      "87/960: Converting Node Type Mul\n",
      "88/960: Converting Node Type MatMul\n",
      "89/960: Converting Node Type Add\n",
      "90/960: Converting Node Type Add\n",
      "91/960: Converting Node Type ReduceMean\n",
      "92/960: Converting Node Type Sub\n",
      "93/960: Converting Node Type Pow\n",
      "94/960: Converting Node Type ReduceMean\n",
      "95/960: Converting Node Type Add\n",
      "96/960: Converting Node Type Sqrt\n",
      "97/960: Converting Node Type Div\n",
      "98/960: Converting Node Type Mul\n",
      "99/960: Converting Node Type Add\n",
      "100/960: Converting Node Type MatMul\n",
      "101/960: Converting Node Type Add\n",
      "102/960: Converting Node Type MatMul\n",
      "103/960: Converting Node Type Add\n",
      "104/960: Converting Node Type MatMul\n",
      "105/960: Converting Node Type Add\n",
      "106/960: Converting Node Type Shape\n",
      "107/960: Converting Node Type Gather\n",
      "108/960: Converting Node Type Shape\n",
      "109/960: Converting Node Type Gather\n",
      "110/960: Converting Node Type Unsqueeze\n",
      "111/960: Converting Node Type Unsqueeze\n",
      "112/960: Converting Node Type Concat\n",
      "113/960: Converting Node Type Reshape\n",
      "114/960: Converting Node Type Transpose\n",
      "115/960: Converting Node Type Shape\n",
      "116/960: Converting Node Type Gather\n",
      "117/960: Converting Node Type Shape\n",
      "118/960: Converting Node Type Gather\n",
      "119/960: Converting Node Type Unsqueeze\n",
      "120/960: Converting Node Type Unsqueeze\n",
      "121/960: Converting Node Type Concat\n",
      "122/960: Converting Node Type Reshape\n",
      "123/960: Converting Node Type Shape\n",
      "124/960: Converting Node Type Gather\n",
      "125/960: Converting Node Type Shape\n",
      "126/960: Converting Node Type Gather\n",
      "127/960: Converting Node Type Unsqueeze\n",
      "128/960: Converting Node Type Unsqueeze\n",
      "129/960: Converting Node Type Concat\n",
      "130/960: Converting Node Type Reshape\n",
      "131/960: Converting Node Type Transpose\n",
      "132/960: Converting Node Type Transpose\n",
      "133/960: Converting Node Type MatMul\n",
      "134/960: Converting Node Type Div\n",
      "135/960: Converting Node Type Add\n",
      "136/960: Converting Node Type Softmax\n",
      "137/960: Converting Node Type MatMul\n",
      "138/960: Converting Node Type Transpose\n",
      "139/960: Converting Node Type Shape\n",
      "140/960: Converting Node Type Gather\n",
      "141/960: Converting Node Type Shape\n",
      "142/960: Converting Node Type Gather\n",
      "143/960: Converting Node Type Unsqueeze\n",
      "144/960: Converting Node Type Unsqueeze\n",
      "145/960: Converting Node Type Concat\n",
      "146/960: Converting Node Type Reshape\n",
      "147/960: Converting Node Type MatMul\n",
      "148/960: Converting Node Type Add\n",
      "149/960: Converting Node Type Add\n",
      "150/960: Converting Node Type ReduceMean\n",
      "151/960: Converting Node Type Sub\n",
      "152/960: Converting Node Type Pow\n",
      "153/960: Converting Node Type ReduceMean\n",
      "154/960: Converting Node Type Add\n",
      "155/960: Converting Node Type Sqrt\n",
      "156/960: Converting Node Type Div\n",
      "157/960: Converting Node Type Mul\n",
      "158/960: Converting Node Type Add\n",
      "159/960: Converting Node Type MatMul\n",
      "160/960: Converting Node Type Add\n",
      "161/960: Converting Node Type Mul\n",
      "162/960: Converting Node Type Div\n",
      "163/960: Converting Node Type Erf\n",
      "164/960: Converting Node Type Add\n",
      "165/960: Converting Node Type Mul\n",
      "166/960: Converting Node Type MatMul\n",
      "167/960: Converting Node Type Add\n",
      "168/960: Converting Node Type Add\n",
      "169/960: Converting Node Type ReduceMean\n",
      "170/960: Converting Node Type Sub\n",
      "171/960: Converting Node Type Pow\n",
      "172/960: Converting Node Type ReduceMean\n",
      "173/960: Converting Node Type Add\n",
      "174/960: Converting Node Type Sqrt\n",
      "175/960: Converting Node Type Div\n",
      "176/960: Converting Node Type Mul\n",
      "177/960: Converting Node Type Add\n",
      "178/960: Converting Node Type MatMul\n",
      "179/960: Converting Node Type Add\n",
      "180/960: Converting Node Type MatMul\n",
      "181/960: Converting Node Type Add\n",
      "182/960: Converting Node Type MatMul\n",
      "183/960: Converting Node Type Add\n",
      "184/960: Converting Node Type Shape\n",
      "185/960: Converting Node Type Gather\n",
      "186/960: Converting Node Type Shape\n",
      "187/960: Converting Node Type Gather\n",
      "188/960: Converting Node Type Unsqueeze\n",
      "189/960: Converting Node Type Unsqueeze\n",
      "190/960: Converting Node Type Concat\n",
      "191/960: Converting Node Type Reshape\n",
      "192/960: Converting Node Type Transpose\n",
      "193/960: Converting Node Type Shape\n",
      "194/960: Converting Node Type Gather\n",
      "195/960: Converting Node Type Shape\n",
      "196/960: Converting Node Type Gather\n",
      "197/960: Converting Node Type Unsqueeze\n",
      "198/960: Converting Node Type Unsqueeze\n",
      "199/960: Converting Node Type Concat\n",
      "200/960: Converting Node Type Reshape\n",
      "201/960: Converting Node Type Shape\n",
      "202/960: Converting Node Type Gather\n",
      "203/960: Converting Node Type Shape\n",
      "204/960: Converting Node Type Gather\n",
      "205/960: Converting Node Type Unsqueeze\n",
      "206/960: Converting Node Type Unsqueeze\n",
      "207/960: Converting Node Type Concat\n",
      "208/960: Converting Node Type Reshape\n",
      "209/960: Converting Node Type Transpose\n",
      "210/960: Converting Node Type Transpose\n",
      "211/960: Converting Node Type MatMul\n",
      "212/960: Converting Node Type Div\n",
      "213/960: Converting Node Type Add\n",
      "214/960: Converting Node Type Softmax\n",
      "215/960: Converting Node Type MatMul\n",
      "216/960: Converting Node Type Transpose\n",
      "217/960: Converting Node Type Shape\n",
      "218/960: Converting Node Type Gather\n",
      "219/960: Converting Node Type Shape\n",
      "220/960: Converting Node Type Gather\n",
      "221/960: Converting Node Type Unsqueeze\n",
      "222/960: Converting Node Type Unsqueeze\n",
      "223/960: Converting Node Type Concat\n",
      "224/960: Converting Node Type Reshape\n",
      "225/960: Converting Node Type MatMul\n",
      "226/960: Converting Node Type Add\n",
      "227/960: Converting Node Type Add\n",
      "228/960: Converting Node Type ReduceMean\n",
      "229/960: Converting Node Type Sub\n",
      "230/960: Converting Node Type Pow\n",
      "231/960: Converting Node Type ReduceMean\n",
      "232/960: Converting Node Type Add\n",
      "233/960: Converting Node Type Sqrt\n",
      "234/960: Converting Node Type Div\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "235/960: Converting Node Type Mul\n",
      "236/960: Converting Node Type Add\n",
      "237/960: Converting Node Type MatMul\n",
      "238/960: Converting Node Type Add\n",
      "239/960: Converting Node Type Mul\n",
      "240/960: Converting Node Type Div\n",
      "241/960: Converting Node Type Erf\n",
      "242/960: Converting Node Type Add\n",
      "243/960: Converting Node Type Mul\n",
      "244/960: Converting Node Type MatMul\n",
      "245/960: Converting Node Type Add\n",
      "246/960: Converting Node Type Add\n",
      "247/960: Converting Node Type ReduceMean\n",
      "248/960: Converting Node Type Sub\n",
      "249/960: Converting Node Type Pow\n",
      "250/960: Converting Node Type ReduceMean\n",
      "251/960: Converting Node Type Add\n",
      "252/960: Converting Node Type Sqrt\n",
      "253/960: Converting Node Type Div\n",
      "254/960: Converting Node Type Mul\n",
      "255/960: Converting Node Type Add\n",
      "256/960: Converting Node Type MatMul\n",
      "257/960: Converting Node Type Add\n",
      "258/960: Converting Node Type MatMul\n",
      "259/960: Converting Node Type Add\n",
      "260/960: Converting Node Type MatMul\n",
      "261/960: Converting Node Type Add\n",
      "262/960: Converting Node Type Shape\n",
      "263/960: Converting Node Type Gather\n",
      "264/960: Converting Node Type Shape\n",
      "265/960: Converting Node Type Gather\n",
      "266/960: Converting Node Type Unsqueeze\n",
      "267/960: Converting Node Type Unsqueeze\n",
      "268/960: Converting Node Type Concat\n",
      "269/960: Converting Node Type Reshape\n",
      "270/960: Converting Node Type Transpose\n",
      "271/960: Converting Node Type Shape\n",
      "272/960: Converting Node Type Gather\n",
      "273/960: Converting Node Type Shape\n",
      "274/960: Converting Node Type Gather\n",
      "275/960: Converting Node Type Unsqueeze\n",
      "276/960: Converting Node Type Unsqueeze\n",
      "277/960: Converting Node Type Concat\n",
      "278/960: Converting Node Type Reshape\n",
      "279/960: Converting Node Type Shape\n",
      "280/960: Converting Node Type Gather\n",
      "281/960: Converting Node Type Shape\n",
      "282/960: Converting Node Type Gather\n",
      "283/960: Converting Node Type Unsqueeze\n",
      "284/960: Converting Node Type Unsqueeze\n",
      "285/960: Converting Node Type Concat\n",
      "286/960: Converting Node Type Reshape\n",
      "287/960: Converting Node Type Transpose\n",
      "288/960: Converting Node Type Transpose\n",
      "289/960: Converting Node Type MatMul\n",
      "290/960: Converting Node Type Div\n",
      "291/960: Converting Node Type Add\n",
      "292/960: Converting Node Type Softmax\n",
      "293/960: Converting Node Type MatMul\n",
      "294/960: Converting Node Type Transpose\n",
      "295/960: Converting Node Type Shape\n",
      "296/960: Converting Node Type Gather\n",
      "297/960: Converting Node Type Shape\n",
      "298/960: Converting Node Type Gather\n",
      "299/960: Converting Node Type Unsqueeze\n",
      "300/960: Converting Node Type Unsqueeze\n",
      "301/960: Converting Node Type Concat\n",
      "302/960: Converting Node Type Reshape\n",
      "303/960: Converting Node Type MatMul\n",
      "304/960: Converting Node Type Add\n",
      "305/960: Converting Node Type Add\n",
      "306/960: Converting Node Type ReduceMean\n",
      "307/960: Converting Node Type Sub\n",
      "308/960: Converting Node Type Pow\n",
      "309/960: Converting Node Type ReduceMean\n",
      "310/960: Converting Node Type Add\n",
      "311/960: Converting Node Type Sqrt\n",
      "312/960: Converting Node Type Div\n",
      "313/960: Converting Node Type Mul\n",
      "314/960: Converting Node Type Add\n",
      "315/960: Converting Node Type MatMul\n",
      "316/960: Converting Node Type Add\n",
      "317/960: Converting Node Type Mul\n",
      "318/960: Converting Node Type Div\n",
      "319/960: Converting Node Type Erf\n",
      "320/960: Converting Node Type Add\n",
      "321/960: Converting Node Type Mul\n",
      "322/960: Converting Node Type MatMul\n",
      "323/960: Converting Node Type Add\n",
      "324/960: Converting Node Type Add\n",
      "325/960: Converting Node Type ReduceMean\n",
      "326/960: Converting Node Type Sub\n",
      "327/960: Converting Node Type Pow\n",
      "328/960: Converting Node Type ReduceMean\n",
      "329/960: Converting Node Type Add\n",
      "330/960: Converting Node Type Sqrt\n",
      "331/960: Converting Node Type Div\n",
      "332/960: Converting Node Type Mul\n",
      "333/960: Converting Node Type Add\n",
      "334/960: Converting Node Type MatMul\n",
      "335/960: Converting Node Type Add\n",
      "336/960: Converting Node Type MatMul\n",
      "337/960: Converting Node Type Add\n",
      "338/960: Converting Node Type MatMul\n",
      "339/960: Converting Node Type Add\n",
      "340/960: Converting Node Type Shape\n",
      "341/960: Converting Node Type Gather\n",
      "342/960: Converting Node Type Shape\n",
      "343/960: Converting Node Type Gather\n",
      "344/960: Converting Node Type Unsqueeze\n",
      "345/960: Converting Node Type Unsqueeze\n",
      "346/960: Converting Node Type Concat\n",
      "347/960: Converting Node Type Reshape\n",
      "348/960: Converting Node Type Transpose\n",
      "349/960: Converting Node Type Shape\n",
      "350/960: Converting Node Type Gather\n",
      "351/960: Converting Node Type Shape\n",
      "352/960: Converting Node Type Gather\n",
      "353/960: Converting Node Type Unsqueeze\n",
      "354/960: Converting Node Type Unsqueeze\n",
      "355/960: Converting Node Type Concat\n",
      "356/960: Converting Node Type Reshape\n",
      "357/960: Converting Node Type Shape\n",
      "358/960: Converting Node Type Gather\n",
      "359/960: Converting Node Type Shape\n",
      "360/960: Converting Node Type Gather\n",
      "361/960: Converting Node Type Unsqueeze\n",
      "362/960: Converting Node Type Unsqueeze\n",
      "363/960: Converting Node Type Concat\n",
      "364/960: Converting Node Type Reshape\n",
      "365/960: Converting Node Type Transpose\n",
      "366/960: Converting Node Type Transpose\n",
      "367/960: Converting Node Type MatMul\n",
      "368/960: Converting Node Type Div\n",
      "369/960: Converting Node Type Add\n",
      "370/960: Converting Node Type Softmax\n",
      "371/960: Converting Node Type MatMul\n",
      "372/960: Converting Node Type Transpose\n",
      "373/960: Converting Node Type Shape\n",
      "374/960: Converting Node Type Gather\n",
      "375/960: Converting Node Type Shape\n",
      "376/960: Converting Node Type Gather\n",
      "377/960: Converting Node Type Unsqueeze\n",
      "378/960: Converting Node Type Unsqueeze\n",
      "379/960: Converting Node Type Concat\n",
      "380/960: Converting Node Type Reshape\n",
      "381/960: Converting Node Type MatMul\n",
      "382/960: Converting Node Type Add\n",
      "383/960: Converting Node Type Add\n",
      "384/960: Converting Node Type ReduceMean\n",
      "385/960: Converting Node Type Sub\n",
      "386/960: Converting Node Type Pow\n",
      "387/960: Converting Node Type ReduceMean\n",
      "388/960: Converting Node Type Add\n",
      "389/960: Converting Node Type Sqrt\n",
      "390/960: Converting Node Type Div\n",
      "391/960: Converting Node Type Mul\n",
      "392/960: Converting Node Type Add\n",
      "393/960: Converting Node Type MatMul\n",
      "394/960: Converting Node Type Add\n",
      "395/960: Converting Node Type Mul\n",
      "396/960: Converting Node Type Div\n",
      "397/960: Converting Node Type Erf\n",
      "398/960: Converting Node Type Add\n",
      "399/960: Converting Node Type Mul\n",
      "400/960: Converting Node Type MatMul\n",
      "401/960: Converting Node Type Add\n",
      "402/960: Converting Node Type Add\n",
      "403/960: Converting Node Type ReduceMean\n",
      "404/960: Converting Node Type Sub\n",
      "405/960: Converting Node Type Pow\n",
      "406/960: Converting Node Type ReduceMean\n",
      "407/960: Converting Node Type Add\n",
      "408/960: Converting Node Type Sqrt\n",
      "409/960: Converting Node Type Div\n",
      "410/960: Converting Node Type Mul\n",
      "411/960: Converting Node Type Add\n",
      "412/960: Converting Node Type MatMul\n",
      "413/960: Converting Node Type Add\n",
      "414/960: Converting Node Type MatMul\n",
      "415/960: Converting Node Type Add\n",
      "416/960: Converting Node Type MatMul\n",
      "417/960: Converting Node Type Add\n",
      "418/960: Converting Node Type Shape\n",
      "419/960: Converting Node Type Gather\n",
      "420/960: Converting Node Type Shape\n",
      "421/960: Converting Node Type Gather\n",
      "422/960: Converting Node Type Unsqueeze\n",
      "423/960: Converting Node Type Unsqueeze\n",
      "424/960: Converting Node Type Concat\n",
      "425/960: Converting Node Type Reshape\n",
      "426/960: Converting Node Type Transpose\n",
      "427/960: Converting Node Type Shape\n",
      "428/960: Converting Node Type Gather\n",
      "429/960: Converting Node Type Shape\n",
      "430/960: Converting Node Type Gather\n",
      "431/960: Converting Node Type Unsqueeze\n",
      "432/960: Converting Node Type Unsqueeze\n",
      "433/960: Converting Node Type Concat\n",
      "434/960: Converting Node Type Reshape\n",
      "435/960: Converting Node Type Shape\n",
      "436/960: Converting Node Type Gather\n",
      "437/960: Converting Node Type Shape\n",
      "438/960: Converting Node Type Gather\n",
      "439/960: Converting Node Type Unsqueeze\n",
      "440/960: Converting Node Type Unsqueeze\n",
      "441/960: Converting Node Type Concat\n",
      "442/960: Converting Node Type Reshape\n",
      "443/960: Converting Node Type Transpose\n",
      "444/960: Converting Node Type Transpose\n",
      "445/960: Converting Node Type MatMul\n",
      "446/960: Converting Node Type Div\n",
      "447/960: Converting Node Type Add\n",
      "448/960: Converting Node Type Softmax\n",
      "449/960: Converting Node Type MatMul\n",
      "450/960: Converting Node Type Transpose\n",
      "451/960: Converting Node Type Shape\n",
      "452/960: Converting Node Type Gather\n",
      "453/960: Converting Node Type Shape\n",
      "454/960: Converting Node Type Gather\n",
      "455/960: Converting Node Type Unsqueeze\n",
      "456/960: Converting Node Type Unsqueeze\n",
      "457/960: Converting Node Type Concat\n",
      "458/960: Converting Node Type Reshape\n",
      "459/960: Converting Node Type MatMul\n",
      "460/960: Converting Node Type Add\n",
      "461/960: Converting Node Type Add\n",
      "462/960: Converting Node Type ReduceMean\n",
      "463/960: Converting Node Type Sub\n",
      "464/960: Converting Node Type Pow\n",
      "465/960: Converting Node Type ReduceMean\n",
      "466/960: Converting Node Type Add\n",
      "467/960: Converting Node Type Sqrt\n",
      "468/960: Converting Node Type Div\n",
      "469/960: Converting Node Type Mul\n",
      "470/960: Converting Node Type Add\n",
      "471/960: Converting Node Type MatMul\n",
      "472/960: Converting Node Type Add\n",
      "473/960: Converting Node Type Mul\n",
      "474/960: Converting Node Type Div\n",
      "475/960: Converting Node Type Erf\n",
      "476/960: Converting Node Type Add\n",
      "477/960: Converting Node Type Mul\n",
      "478/960: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "479/960: Converting Node Type Add\n",
      "480/960: Converting Node Type Add\n",
      "481/960: Converting Node Type ReduceMean\n",
      "482/960: Converting Node Type Sub\n",
      "483/960: Converting Node Type Pow\n",
      "484/960: Converting Node Type ReduceMean\n",
      "485/960: Converting Node Type Add\n",
      "486/960: Converting Node Type Sqrt\n",
      "487/960: Converting Node Type Div\n",
      "488/960: Converting Node Type Mul\n",
      "489/960: Converting Node Type Add\n",
      "490/960: Converting Node Type MatMul\n",
      "491/960: Converting Node Type Add\n",
      "492/960: Converting Node Type MatMul\n",
      "493/960: Converting Node Type Add\n",
      "494/960: Converting Node Type MatMul\n",
      "495/960: Converting Node Type Add\n",
      "496/960: Converting Node Type Shape\n",
      "497/960: Converting Node Type Gather\n",
      "498/960: Converting Node Type Shape\n",
      "499/960: Converting Node Type Gather\n",
      "500/960: Converting Node Type Unsqueeze\n",
      "501/960: Converting Node Type Unsqueeze\n",
      "502/960: Converting Node Type Concat\n",
      "503/960: Converting Node Type Reshape\n",
      "504/960: Converting Node Type Transpose\n",
      "505/960: Converting Node Type Shape\n",
      "506/960: Converting Node Type Gather\n",
      "507/960: Converting Node Type Shape\n",
      "508/960: Converting Node Type Gather\n",
      "509/960: Converting Node Type Unsqueeze\n",
      "510/960: Converting Node Type Unsqueeze\n",
      "511/960: Converting Node Type Concat\n",
      "512/960: Converting Node Type Reshape\n",
      "513/960: Converting Node Type Shape\n",
      "514/960: Converting Node Type Gather\n",
      "515/960: Converting Node Type Shape\n",
      "516/960: Converting Node Type Gather\n",
      "517/960: Converting Node Type Unsqueeze\n",
      "518/960: Converting Node Type Unsqueeze\n",
      "519/960: Converting Node Type Concat\n",
      "520/960: Converting Node Type Reshape\n",
      "521/960: Converting Node Type Transpose\n",
      "522/960: Converting Node Type Transpose\n",
      "523/960: Converting Node Type MatMul\n",
      "524/960: Converting Node Type Div\n",
      "525/960: Converting Node Type Add\n",
      "526/960: Converting Node Type Softmax\n",
      "527/960: Converting Node Type MatMul\n",
      "528/960: Converting Node Type Transpose\n",
      "529/960: Converting Node Type Shape\n",
      "530/960: Converting Node Type Gather\n",
      "531/960: Converting Node Type Shape\n",
      "532/960: Converting Node Type Gather\n",
      "533/960: Converting Node Type Unsqueeze\n",
      "534/960: Converting Node Type Unsqueeze\n",
      "535/960: Converting Node Type Concat\n",
      "536/960: Converting Node Type Reshape\n",
      "537/960: Converting Node Type MatMul\n",
      "538/960: Converting Node Type Add\n",
      "539/960: Converting Node Type Add\n",
      "540/960: Converting Node Type ReduceMean\n",
      "541/960: Converting Node Type Sub\n",
      "542/960: Converting Node Type Pow\n",
      "543/960: Converting Node Type ReduceMean\n",
      "544/960: Converting Node Type Add\n",
      "545/960: Converting Node Type Sqrt\n",
      "546/960: Converting Node Type Div\n",
      "547/960: Converting Node Type Mul\n",
      "548/960: Converting Node Type Add\n",
      "549/960: Converting Node Type MatMul\n",
      "550/960: Converting Node Type Add\n",
      "551/960: Converting Node Type Mul\n",
      "552/960: Converting Node Type Div\n",
      "553/960: Converting Node Type Erf\n",
      "554/960: Converting Node Type Add\n",
      "555/960: Converting Node Type Mul\n",
      "556/960: Converting Node Type MatMul\n",
      "557/960: Converting Node Type Add\n",
      "558/960: Converting Node Type Add\n",
      "559/960: Converting Node Type ReduceMean\n",
      "560/960: Converting Node Type Sub\n",
      "561/960: Converting Node Type Pow\n",
      "562/960: Converting Node Type ReduceMean\n",
      "563/960: Converting Node Type Add\n",
      "564/960: Converting Node Type Sqrt\n",
      "565/960: Converting Node Type Div\n",
      "566/960: Converting Node Type Mul\n",
      "567/960: Converting Node Type Add\n",
      "568/960: Converting Node Type MatMul\n",
      "569/960: Converting Node Type Add\n",
      "570/960: Converting Node Type MatMul\n",
      "571/960: Converting Node Type Add\n",
      "572/960: Converting Node Type MatMul\n",
      "573/960: Converting Node Type Add\n",
      "574/960: Converting Node Type Shape\n",
      "575/960: Converting Node Type Gather\n",
      "576/960: Converting Node Type Shape\n",
      "577/960: Converting Node Type Gather\n",
      "578/960: Converting Node Type Unsqueeze\n",
      "579/960: Converting Node Type Unsqueeze\n",
      "580/960: Converting Node Type Concat\n",
      "581/960: Converting Node Type Reshape\n",
      "582/960: Converting Node Type Transpose\n",
      "583/960: Converting Node Type Shape\n",
      "584/960: Converting Node Type Gather\n",
      "585/960: Converting Node Type Shape\n",
      "586/960: Converting Node Type Gather\n",
      "587/960: Converting Node Type Unsqueeze\n",
      "588/960: Converting Node Type Unsqueeze\n",
      "589/960: Converting Node Type Concat\n",
      "590/960: Converting Node Type Reshape\n",
      "591/960: Converting Node Type Shape\n",
      "592/960: Converting Node Type Gather\n",
      "593/960: Converting Node Type Shape\n",
      "594/960: Converting Node Type Gather\n",
      "595/960: Converting Node Type Unsqueeze\n",
      "596/960: Converting Node Type Unsqueeze\n",
      "597/960: Converting Node Type Concat\n",
      "598/960: Converting Node Type Reshape\n",
      "599/960: Converting Node Type Transpose\n",
      "600/960: Converting Node Type Transpose\n",
      "601/960: Converting Node Type MatMul\n",
      "602/960: Converting Node Type Div\n",
      "603/960: Converting Node Type Add\n",
      "604/960: Converting Node Type Softmax\n",
      "605/960: Converting Node Type MatMul\n",
      "606/960: Converting Node Type Transpose\n",
      "607/960: Converting Node Type Shape\n",
      "608/960: Converting Node Type Gather\n",
      "609/960: Converting Node Type Shape\n",
      "610/960: Converting Node Type Gather\n",
      "611/960: Converting Node Type Unsqueeze\n",
      "612/960: Converting Node Type Unsqueeze\n",
      "613/960: Converting Node Type Concat\n",
      "614/960: Converting Node Type Reshape\n",
      "615/960: Converting Node Type MatMul\n",
      "616/960: Converting Node Type Add\n",
      "617/960: Converting Node Type Add\n",
      "618/960: Converting Node Type ReduceMean\n",
      "619/960: Converting Node Type Sub\n",
      "620/960: Converting Node Type Pow\n",
      "621/960: Converting Node Type ReduceMean\n",
      "622/960: Converting Node Type Add\n",
      "623/960: Converting Node Type Sqrt\n",
      "624/960: Converting Node Type Div\n",
      "625/960: Converting Node Type Mul\n",
      "626/960: Converting Node Type Add\n",
      "627/960: Converting Node Type MatMul\n",
      "628/960: Converting Node Type Add\n",
      "629/960: Converting Node Type Mul\n",
      "630/960: Converting Node Type Div\n",
      "631/960: Converting Node Type Erf\n",
      "632/960: Converting Node Type Add\n",
      "633/960: Converting Node Type Mul\n",
      "634/960: Converting Node Type MatMul\n",
      "635/960: Converting Node Type Add\n",
      "636/960: Converting Node Type Add\n",
      "637/960: Converting Node Type ReduceMean\n",
      "638/960: Converting Node Type Sub\n",
      "639/960: Converting Node Type Pow\n",
      "640/960: Converting Node Type ReduceMean\n",
      "641/960: Converting Node Type Add\n",
      "642/960: Converting Node Type Sqrt\n",
      "643/960: Converting Node Type Div\n",
      "644/960: Converting Node Type Mul\n",
      "645/960: Converting Node Type Add\n",
      "646/960: Converting Node Type MatMul\n",
      "647/960: Converting Node Type Add\n",
      "648/960: Converting Node Type MatMul\n",
      "649/960: Converting Node Type Add\n",
      "650/960: Converting Node Type MatMul\n",
      "651/960: Converting Node Type Add\n",
      "652/960: Converting Node Type Shape\n",
      "653/960: Converting Node Type Gather\n",
      "654/960: Converting Node Type Shape\n",
      "655/960: Converting Node Type Gather\n",
      "656/960: Converting Node Type Unsqueeze\n",
      "657/960: Converting Node Type Unsqueeze\n",
      "658/960: Converting Node Type Concat\n",
      "659/960: Converting Node Type Reshape\n",
      "660/960: Converting Node Type Transpose\n",
      "661/960: Converting Node Type Shape\n",
      "662/960: Converting Node Type Gather\n",
      "663/960: Converting Node Type Shape\n",
      "664/960: Converting Node Type Gather\n",
      "665/960: Converting Node Type Unsqueeze\n",
      "666/960: Converting Node Type Unsqueeze\n",
      "667/960: Converting Node Type Concat\n",
      "668/960: Converting Node Type Reshape\n",
      "669/960: Converting Node Type Shape\n",
      "670/960: Converting Node Type Gather\n",
      "671/960: Converting Node Type Shape\n",
      "672/960: Converting Node Type Gather\n",
      "673/960: Converting Node Type Unsqueeze\n",
      "674/960: Converting Node Type Unsqueeze\n",
      "675/960: Converting Node Type Concat\n",
      "676/960: Converting Node Type Reshape\n",
      "677/960: Converting Node Type Transpose\n",
      "678/960: Converting Node Type Transpose\n",
      "679/960: Converting Node Type MatMul\n",
      "680/960: Converting Node Type Div\n",
      "681/960: Converting Node Type Add\n",
      "682/960: Converting Node Type Softmax\n",
      "683/960: Converting Node Type MatMul\n",
      "684/960: Converting Node Type Transpose\n",
      "685/960: Converting Node Type Shape\n",
      "686/960: Converting Node Type Gather\n",
      "687/960: Converting Node Type Shape\n",
      "688/960: Converting Node Type Gather\n",
      "689/960: Converting Node Type Unsqueeze\n",
      "690/960: Converting Node Type Unsqueeze\n",
      "691/960: Converting Node Type Concat\n",
      "692/960: Converting Node Type Reshape\n",
      "693/960: Converting Node Type MatMul\n",
      "694/960: Converting Node Type Add\n",
      "695/960: Converting Node Type Add\n",
      "696/960: Converting Node Type ReduceMean\n",
      "697/960: Converting Node Type Sub\n",
      "698/960: Converting Node Type Pow\n",
      "699/960: Converting Node Type ReduceMean\n",
      "700/960: Converting Node Type Add\n",
      "701/960: Converting Node Type Sqrt\n",
      "702/960: Converting Node Type Div\n",
      "703/960: Converting Node Type Mul\n",
      "704/960: Converting Node Type Add\n",
      "705/960: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "706/960: Converting Node Type Add\n",
      "707/960: Converting Node Type Mul\n",
      "708/960: Converting Node Type Div\n",
      "709/960: Converting Node Type Erf\n",
      "710/960: Converting Node Type Add\n",
      "711/960: Converting Node Type Mul\n",
      "712/960: Converting Node Type MatMul\n",
      "713/960: Converting Node Type Add\n",
      "714/960: Converting Node Type Add\n",
      "715/960: Converting Node Type ReduceMean\n",
      "716/960: Converting Node Type Sub\n",
      "717/960: Converting Node Type Pow\n",
      "718/960: Converting Node Type ReduceMean\n",
      "719/960: Converting Node Type Add\n",
      "720/960: Converting Node Type Sqrt\n",
      "721/960: Converting Node Type Div\n",
      "722/960: Converting Node Type Mul\n",
      "723/960: Converting Node Type Add\n",
      "724/960: Converting Node Type MatMul\n",
      "725/960: Converting Node Type Add\n",
      "726/960: Converting Node Type MatMul\n",
      "727/960: Converting Node Type Add\n",
      "728/960: Converting Node Type MatMul\n",
      "729/960: Converting Node Type Add\n",
      "730/960: Converting Node Type Shape\n",
      "731/960: Converting Node Type Gather\n",
      "732/960: Converting Node Type Shape\n",
      "733/960: Converting Node Type Gather\n",
      "734/960: Converting Node Type Unsqueeze\n",
      "735/960: Converting Node Type Unsqueeze\n",
      "736/960: Converting Node Type Concat\n",
      "737/960: Converting Node Type Reshape\n",
      "738/960: Converting Node Type Transpose\n",
      "739/960: Converting Node Type Shape\n",
      "740/960: Converting Node Type Gather\n",
      "741/960: Converting Node Type Shape\n",
      "742/960: Converting Node Type Gather\n",
      "743/960: Converting Node Type Unsqueeze\n",
      "744/960: Converting Node Type Unsqueeze\n",
      "745/960: Converting Node Type Concat\n",
      "746/960: Converting Node Type Reshape\n",
      "747/960: Converting Node Type Shape\n",
      "748/960: Converting Node Type Gather\n",
      "749/960: Converting Node Type Shape\n",
      "750/960: Converting Node Type Gather\n",
      "751/960: Converting Node Type Unsqueeze\n",
      "752/960: Converting Node Type Unsqueeze\n",
      "753/960: Converting Node Type Concat\n",
      "754/960: Converting Node Type Reshape\n",
      "755/960: Converting Node Type Transpose\n",
      "756/960: Converting Node Type Transpose\n",
      "757/960: Converting Node Type MatMul\n",
      "758/960: Converting Node Type Div\n",
      "759/960: Converting Node Type Add\n",
      "760/960: Converting Node Type Softmax\n",
      "761/960: Converting Node Type MatMul\n",
      "762/960: Converting Node Type Transpose\n",
      "763/960: Converting Node Type Shape\n",
      "764/960: Converting Node Type Gather\n",
      "765/960: Converting Node Type Shape\n",
      "766/960: Converting Node Type Gather\n",
      "767/960: Converting Node Type Unsqueeze\n",
      "768/960: Converting Node Type Unsqueeze\n",
      "769/960: Converting Node Type Concat\n",
      "770/960: Converting Node Type Reshape\n",
      "771/960: Converting Node Type MatMul\n",
      "772/960: Converting Node Type Add\n",
      "773/960: Converting Node Type Add\n",
      "774/960: Converting Node Type ReduceMean\n",
      "775/960: Converting Node Type Sub\n",
      "776/960: Converting Node Type Pow\n",
      "777/960: Converting Node Type ReduceMean\n",
      "778/960: Converting Node Type Add\n",
      "779/960: Converting Node Type Sqrt\n",
      "780/960: Converting Node Type Div\n",
      "781/960: Converting Node Type Mul\n",
      "782/960: Converting Node Type Add\n",
      "783/960: Converting Node Type MatMul\n",
      "784/960: Converting Node Type Add\n",
      "785/960: Converting Node Type Mul\n",
      "786/960: Converting Node Type Div\n",
      "787/960: Converting Node Type Erf\n",
      "788/960: Converting Node Type Add\n",
      "789/960: Converting Node Type Mul\n",
      "790/960: Converting Node Type MatMul\n",
      "791/960: Converting Node Type Add\n",
      "792/960: Converting Node Type Add\n",
      "793/960: Converting Node Type ReduceMean\n",
      "794/960: Converting Node Type Sub\n",
      "795/960: Converting Node Type Pow\n",
      "796/960: Converting Node Type ReduceMean\n",
      "797/960: Converting Node Type Add\n",
      "798/960: Converting Node Type Sqrt\n",
      "799/960: Converting Node Type Div\n",
      "800/960: Converting Node Type Mul\n",
      "801/960: Converting Node Type Add\n",
      "802/960: Converting Node Type MatMul\n",
      "803/960: Converting Node Type Add\n",
      "804/960: Converting Node Type MatMul\n",
      "805/960: Converting Node Type Add\n",
      "806/960: Converting Node Type MatMul\n",
      "807/960: Converting Node Type Add\n",
      "808/960: Converting Node Type Shape\n",
      "809/960: Converting Node Type Gather\n",
      "810/960: Converting Node Type Shape\n",
      "811/960: Converting Node Type Gather\n",
      "812/960: Converting Node Type Unsqueeze\n",
      "813/960: Converting Node Type Unsqueeze\n",
      "814/960: Converting Node Type Concat\n",
      "815/960: Converting Node Type Reshape\n",
      "816/960: Converting Node Type Transpose\n",
      "817/960: Converting Node Type Shape\n",
      "818/960: Converting Node Type Gather\n",
      "819/960: Converting Node Type Shape\n",
      "820/960: Converting Node Type Gather\n",
      "821/960: Converting Node Type Unsqueeze\n",
      "822/960: Converting Node Type Unsqueeze\n",
      "823/960: Converting Node Type Concat\n",
      "824/960: Converting Node Type Reshape\n",
      "825/960: Converting Node Type Shape\n",
      "826/960: Converting Node Type Gather\n",
      "827/960: Converting Node Type Shape\n",
      "828/960: Converting Node Type Gather\n",
      "829/960: Converting Node Type Unsqueeze\n",
      "830/960: Converting Node Type Unsqueeze\n",
      "831/960: Converting Node Type Concat\n",
      "832/960: Converting Node Type Reshape\n",
      "833/960: Converting Node Type Transpose\n",
      "834/960: Converting Node Type Transpose\n",
      "835/960: Converting Node Type MatMul\n",
      "836/960: Converting Node Type Div\n",
      "837/960: Converting Node Type Add\n",
      "838/960: Converting Node Type Softmax\n",
      "839/960: Converting Node Type MatMul\n",
      "840/960: Converting Node Type Transpose\n",
      "841/960: Converting Node Type Shape\n",
      "842/960: Converting Node Type Gather\n",
      "843/960: Converting Node Type Shape\n",
      "844/960: Converting Node Type Gather\n",
      "845/960: Converting Node Type Unsqueeze\n",
      "846/960: Converting Node Type Unsqueeze\n",
      "847/960: Converting Node Type Concat\n",
      "848/960: Converting Node Type Reshape\n",
      "849/960: Converting Node Type MatMul\n",
      "850/960: Converting Node Type Add\n",
      "851/960: Converting Node Type Add\n",
      "852/960: Converting Node Type ReduceMean\n",
      "853/960: Converting Node Type Sub\n",
      "854/960: Converting Node Type Pow\n",
      "855/960: Converting Node Type ReduceMean\n",
      "856/960: Converting Node Type Add\n",
      "857/960: Converting Node Type Sqrt\n",
      "858/960: Converting Node Type Div\n",
      "859/960: Converting Node Type Mul\n",
      "860/960: Converting Node Type Add\n",
      "861/960: Converting Node Type MatMul\n",
      "862/960: Converting Node Type Add\n",
      "863/960: Converting Node Type Mul\n",
      "864/960: Converting Node Type Div\n",
      "865/960: Converting Node Type Erf\n",
      "866/960: Converting Node Type Add\n",
      "867/960: Converting Node Type Mul\n",
      "868/960: Converting Node Type MatMul\n",
      "869/960: Converting Node Type Add\n",
      "870/960: Converting Node Type Add\n",
      "871/960: Converting Node Type ReduceMean\n",
      "872/960: Converting Node Type Sub\n",
      "873/960: Converting Node Type Pow\n",
      "874/960: Converting Node Type ReduceMean\n",
      "875/960: Converting Node Type Add\n",
      "876/960: Converting Node Type Sqrt\n",
      "877/960: Converting Node Type Div\n",
      "878/960: Converting Node Type Mul\n",
      "879/960: Converting Node Type Add\n",
      "880/960: Converting Node Type MatMul\n",
      "881/960: Converting Node Type Add\n",
      "882/960: Converting Node Type MatMul\n",
      "883/960: Converting Node Type Add\n",
      "884/960: Converting Node Type MatMul\n",
      "885/960: Converting Node Type Add\n",
      "886/960: Converting Node Type Shape\n",
      "887/960: Converting Node Type Gather\n",
      "888/960: Converting Node Type Shape\n",
      "889/960: Converting Node Type Gather\n",
      "890/960: Converting Node Type Unsqueeze\n",
      "891/960: Converting Node Type Unsqueeze\n",
      "892/960: Converting Node Type Concat\n",
      "893/960: Converting Node Type Reshape\n",
      "894/960: Converting Node Type Transpose\n",
      "895/960: Converting Node Type Shape\n",
      "896/960: Converting Node Type Gather\n",
      "897/960: Converting Node Type Shape\n",
      "898/960: Converting Node Type Gather\n",
      "899/960: Converting Node Type Unsqueeze\n",
      "900/960: Converting Node Type Unsqueeze\n",
      "901/960: Converting Node Type Concat\n",
      "902/960: Converting Node Type Reshape\n",
      "903/960: Converting Node Type Shape\n",
      "904/960: Converting Node Type Gather\n",
      "905/960: Converting Node Type Shape\n",
      "906/960: Converting Node Type Gather\n",
      "907/960: Converting Node Type Unsqueeze\n",
      "908/960: Converting Node Type Unsqueeze\n",
      "909/960: Converting Node Type Concat\n",
      "910/960: Converting Node Type Reshape\n",
      "911/960: Converting Node Type Transpose\n",
      "912/960: Converting Node Type Transpose\n",
      "913/960: Converting Node Type MatMul\n",
      "914/960: Converting Node Type Div\n",
      "915/960: Converting Node Type Add\n",
      "916/960: Converting Node Type Softmax\n",
      "917/960: Converting Node Type MatMul\n",
      "918/960: Converting Node Type Transpose\n",
      "919/960: Converting Node Type Shape\n",
      "920/960: Converting Node Type Gather\n",
      "921/960: Converting Node Type Shape\n",
      "922/960: Converting Node Type Gather\n",
      "923/960: Converting Node Type Unsqueeze\n",
      "924/960: Converting Node Type Unsqueeze\n",
      "925/960: Converting Node Type Concat\n",
      "926/960: Converting Node Type Reshape\n",
      "927/960: Converting Node Type MatMul\n",
      "928/960: Converting Node Type Add\n",
      "929/960: Converting Node Type Add\n",
      "930/960: Converting Node Type ReduceMean\n",
      "931/960: Converting Node Type Sub\n",
      "932/960: Converting Node Type Pow\n",
      "933/960: Converting Node Type ReduceMean\n",
      "934/960: Converting Node Type Add\n",
      "935/960: Converting Node Type Sqrt\n",
      "936/960: Converting Node Type Div\n",
      "937/960: Converting Node Type Mul\n",
      "938/960: Converting Node Type Add\n",
      "939/960: Converting Node Type MatMul\n",
      "940/960: Converting Node Type Add\n",
      "941/960: Converting Node Type Mul\n",
      "942/960: Converting Node Type Div\n",
      "943/960: Converting Node Type Erf\n",
      "944/960: Converting Node Type Add\n",
      "945/960: Converting Node Type Mul\n",
      "946/960: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "947/960: Converting Node Type Add\n",
      "948/960: Converting Node Type Add\n",
      "949/960: Converting Node Type ReduceMean\n",
      "950/960: Converting Node Type Sub\n",
      "951/960: Converting Node Type Pow\n",
      "952/960: Converting Node Type ReduceMean\n",
      "953/960: Converting Node Type Add\n",
      "954/960: Converting Node Type Sqrt\n",
      "955/960: Converting Node Type Div\n",
      "956/960: Converting Node Type Mul\n",
      "957/960: Converting Node Type Add\n",
      "958/960: Converting Node Type Gather\n",
      "959/960: Converting Node Type Gemm\n",
      "960/960: Converting Node Type Tanh\n",
      "Translation to CoreML spec completed. Now compiling the CoreML model.\n",
      "Model Compilation done.\n"
     ]
    }
   ],
   "source": [
    "# Step 2 - Convert from ONNX to CoreML MLModel\n",
    "mlmodel = convert(model=onnx_model_path, target_ios=\"13\", )\n",
    "mlmodel.save(mlmodel_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Run Prediction on both the models to verify conversion correctness\n",
    "# on given input\n",
    "# PyTorch prediction\n",
    "pred_pt    = model(test_input)\n",
    "\n",
    "# MLModel prediction\n",
    "input_dict = {'input_ids': test_input.numpy().astype(np.float32)}\n",
    "pred_coreml = mlmodel.predict(input_dict, useCPUOnly=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start Scores:  SNR: 116.98051819605858 PSNR: 73.80000517139831\n",
      "End Scores:  SNR: 93.95363293912368 PSNR: 69.97449408105454\n"
     ]
    }
   ],
   "source": [
    "# SNR and PSNR values verification for PyTorch and CoreML MLModel\n",
    "_compute_SNR(pred_pt[0].detach().numpy(), pred_coreml['start_scores'], 'Start Scores: ')\n",
    "_compute_SNR(pred_pt[1].detach().numpy(), pred_coreml['end_scores'], 'End Scores: ')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2 BERT LARGE UNCASED\n",
    "- Please find saved PyTorch, ONNX and CoreML Model https://drive.google.com/drive/u/3/folders/1V4BxddAZ_EzQk18PRSu4lGjAQGqRfxU3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "TMP_DIR = '/tmp/'\n",
    "model_name = 'bert-large-uncased'\n",
    "pt_path = TMP_DIR + model_name + '.pt'\n",
    "onnx_model_path = TMP_DIR + model_name + '.onnx'\n",
    "mlmodel_path  = TMP_DIR + model_name + '.mlmodel'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load BERT Large Model\n",
    "# Details: 24-layer, 1024-hidden, 16-heads, 340M parameters. Trained on lower-cased English text.\n",
    "model = BertModel.from_pretrained(model_name)\n",
    "torch.save(model, pt_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PyTorch to ONNX\n",
    "test_input = torch.randint(0, 512, (1, 512))\n",
    "torch.onnx.export(model,\n",
    "                  test_input,\n",
    "                  onnx_model_path,\n",
    "                  input_names=[\"input_ids\"],\n",
    "                  output_names=[\"start_scores\", \"end_scores\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1/1896: Converting Node Type ConstantOfShape\n",
      "2/1896: Converting Node Type ConstantOfShape\n",
      "3/1896: Converting Node Type Unsqueeze\n",
      "4/1896: Converting Node Type Unsqueeze\n",
      "5/1896: Converting Node Type Sub\n",
      "6/1896: Converting Node Type Mul\n",
      "7/1896: Converting Node Type Expand\n",
      "8/1896: Converting Node Type Gather\n",
      "9/1896: Converting Node Type Gather\n",
      "10/1896: Converting Node Type Gather\n",
      "11/1896: Converting Node Type Add\n",
      "12/1896: Converting Node Type Add\n",
      "13/1896: Converting Node Type ReduceMean\n",
      "14/1896: Converting Node Type Sub\n",
      "15/1896: Converting Node Type Pow\n",
      "16/1896: Converting Node Type ReduceMean\n",
      "17/1896: Converting Node Type Add\n",
      "18/1896: Converting Node Type Sqrt\n",
      "19/1896: Converting Node Type Div\n",
      "20/1896: Converting Node Type Mul\n",
      "21/1896: Converting Node Type Add\n",
      "22/1896: Converting Node Type MatMul\n",
      "23/1896: Converting Node Type Add\n",
      "24/1896: Converting Node Type MatMul\n",
      "25/1896: Converting Node Type Add\n",
      "26/1896: Converting Node Type MatMul\n",
      "27/1896: Converting Node Type Add\n",
      "28/1896: Converting Node Type Shape\n",
      "29/1896: Converting Node Type Gather\n",
      "30/1896: Converting Node Type Shape\n",
      "31/1896: Converting Node Type Gather\n",
      "32/1896: Converting Node Type Unsqueeze\n",
      "33/1896: Converting Node Type Unsqueeze\n",
      "34/1896: Converting Node Type Concat\n",
      "35/1896: Converting Node Type Reshape\n",
      "36/1896: Converting Node Type Transpose\n",
      "37/1896: Converting Node Type Shape\n",
      "38/1896: Converting Node Type Gather\n",
      "39/1896: Converting Node Type Shape\n",
      "40/1896: Converting Node Type Gather\n",
      "41/1896: Converting Node Type Unsqueeze\n",
      "42/1896: Converting Node Type Unsqueeze\n",
      "43/1896: Converting Node Type Concat\n",
      "44/1896: Converting Node Type Reshape\n",
      "45/1896: Converting Node Type Shape\n",
      "46/1896: Converting Node Type Gather\n",
      "47/1896: Converting Node Type Shape\n",
      "48/1896: Converting Node Type Gather\n",
      "49/1896: Converting Node Type Unsqueeze\n",
      "50/1896: Converting Node Type Unsqueeze\n",
      "51/1896: Converting Node Type Concat\n",
      "52/1896: Converting Node Type Reshape\n",
      "53/1896: Converting Node Type Transpose\n",
      "54/1896: Converting Node Type Transpose\n",
      "55/1896: Converting Node Type MatMul\n",
      "56/1896: Converting Node Type Div\n",
      "57/1896: Converting Node Type Add\n",
      "58/1896: Converting Node Type Softmax\n",
      "59/1896: Converting Node Type MatMul\n",
      "60/1896: Converting Node Type Transpose\n",
      "61/1896: Converting Node Type Shape\n",
      "62/1896: Converting Node Type Gather\n",
      "63/1896: Converting Node Type Shape\n",
      "64/1896: Converting Node Type Gather\n",
      "65/1896: Converting Node Type Unsqueeze\n",
      "66/1896: Converting Node Type Unsqueeze\n",
      "67/1896: Converting Node Type Concat\n",
      "68/1896: Converting Node Type Reshape\n",
      "69/1896: Converting Node Type MatMul\n",
      "70/1896: Converting Node Type Add\n",
      "71/1896: Converting Node Type Add\n",
      "72/1896: Converting Node Type ReduceMean\n",
      "73/1896: Converting Node Type Sub\n",
      "74/1896: Converting Node Type Pow\n",
      "75/1896: Converting Node Type ReduceMean\n",
      "76/1896: Converting Node Type Add\n",
      "77/1896: Converting Node Type Sqrt\n",
      "78/1896: Converting Node Type Div\n",
      "79/1896: Converting Node Type Mul\n",
      "80/1896: Converting Node Type Add\n",
      "81/1896: Converting Node Type MatMul\n",
      "82/1896: Converting Node Type Add\n",
      "83/1896: Converting Node Type Mul\n",
      "84/1896: Converting Node Type Div\n",
      "85/1896: Converting Node Type Erf\n",
      "86/1896: Converting Node Type Add\n",
      "87/1896: Converting Node Type Mul\n",
      "88/1896: Converting Node Type MatMul\n",
      "89/1896: Converting Node Type Add\n",
      "90/1896: Converting Node Type Add\n",
      "91/1896: Converting Node Type ReduceMean\n",
      "92/1896: Converting Node Type Sub\n",
      "93/1896: Converting Node Type Pow\n",
      "94/1896: Converting Node Type ReduceMean\n",
      "95/1896: Converting Node Type Add\n",
      "96/1896: Converting Node Type Sqrt\n",
      "97/1896: Converting Node Type Div\n",
      "98/1896: Converting Node Type Mul\n",
      "99/1896: Converting Node Type Add\n",
      "100/1896: Converting Node Type MatMul\n",
      "101/1896: Converting Node Type Add\n",
      "102/1896: Converting Node Type MatMul\n",
      "103/1896: Converting Node Type Add\n",
      "104/1896: Converting Node Type MatMul\n",
      "105/1896: Converting Node Type Add\n",
      "106/1896: Converting Node Type Shape\n",
      "107/1896: Converting Node Type Gather\n",
      "108/1896: Converting Node Type Shape\n",
      "109/1896: Converting Node Type Gather\n",
      "110/1896: Converting Node Type Unsqueeze\n",
      "111/1896: Converting Node Type Unsqueeze\n",
      "112/1896: Converting Node Type Concat\n",
      "113/1896: Converting Node Type Reshape\n",
      "114/1896: Converting Node Type Transpose\n",
      "115/1896: Converting Node Type Shape\n",
      "116/1896: Converting Node Type Gather\n",
      "117/1896: Converting Node Type Shape\n",
      "118/1896: Converting Node Type Gather\n",
      "119/1896: Converting Node Type Unsqueeze\n",
      "120/1896: Converting Node Type Unsqueeze\n",
      "121/1896: Converting Node Type Concat\n",
      "122/1896: Converting Node Type Reshape\n",
      "123/1896: Converting Node Type Shape\n",
      "124/1896: Converting Node Type Gather\n",
      "125/1896: Converting Node Type Shape\n",
      "126/1896: Converting Node Type Gather\n",
      "127/1896: Converting Node Type Unsqueeze\n",
      "128/1896: Converting Node Type Unsqueeze\n",
      "129/1896: Converting Node Type Concat\n",
      "130/1896: Converting Node Type Reshape\n",
      "131/1896: Converting Node Type Transpose\n",
      "132/1896: Converting Node Type Transpose\n",
      "133/1896: Converting Node Type MatMul\n",
      "134/1896: Converting Node Type Div\n",
      "135/1896: Converting Node Type Add\n",
      "136/1896: Converting Node Type Softmax\n",
      "137/1896: Converting Node Type MatMul\n",
      "138/1896: Converting Node Type Transpose\n",
      "139/1896: Converting Node Type Shape\n",
      "140/1896: Converting Node Type Gather\n",
      "141/1896: Converting Node Type Shape\n",
      "142/1896: Converting Node Type Gather\n",
      "143/1896: Converting Node Type Unsqueeze\n",
      "144/1896: Converting Node Type Unsqueeze\n",
      "145/1896: Converting Node Type Concat\n",
      "146/1896: Converting Node Type Reshape\n",
      "147/1896: Converting Node Type MatMul\n",
      "148/1896: Converting Node Type Add\n",
      "149/1896: Converting Node Type Add\n",
      "150/1896: Converting Node Type ReduceMean\n",
      "151/1896: Converting Node Type Sub\n",
      "152/1896: Converting Node Type Pow\n",
      "153/1896: Converting Node Type ReduceMean\n",
      "154/1896: Converting Node Type Add\n",
      "155/1896: Converting Node Type Sqrt\n",
      "156/1896: Converting Node Type Div\n",
      "157/1896: Converting Node Type Mul\n",
      "158/1896: Converting Node Type Add\n",
      "159/1896: Converting Node Type MatMul\n",
      "160/1896: Converting Node Type Add\n",
      "161/1896: Converting Node Type Mul\n",
      "162/1896: Converting Node Type Div\n",
      "163/1896: Converting Node Type Erf\n",
      "164/1896: Converting Node Type Add\n",
      "165/1896: Converting Node Type Mul\n",
      "166/1896: Converting Node Type MatMul\n",
      "167/1896: Converting Node Type Add\n",
      "168/1896: Converting Node Type Add\n",
      "169/1896: Converting Node Type ReduceMean\n",
      "170/1896: Converting Node Type Sub\n",
      "171/1896: Converting Node Type Pow\n",
      "172/1896: Converting Node Type ReduceMean\n",
      "173/1896: Converting Node Type Add\n",
      "174/1896: Converting Node Type Sqrt\n",
      "175/1896: Converting Node Type Div\n",
      "176/1896: Converting Node Type Mul\n",
      "177/1896: Converting Node Type Add\n",
      "178/1896: Converting Node Type MatMul\n",
      "179/1896: Converting Node Type Add\n",
      "180/1896: Converting Node Type MatMul\n",
      "181/1896: Converting Node Type Add\n",
      "182/1896: Converting Node Type MatMul\n",
      "183/1896: Converting Node Type Add\n",
      "184/1896: Converting Node Type Shape\n",
      "185/1896: Converting Node Type Gather\n",
      "186/1896: Converting Node Type Shape\n",
      "187/1896: Converting Node Type Gather\n",
      "188/1896: Converting Node Type Unsqueeze\n",
      "189/1896: Converting Node Type Unsqueeze\n",
      "190/1896: Converting Node Type Concat\n",
      "191/1896: Converting Node Type Reshape\n",
      "192/1896: Converting Node Type Transpose\n",
      "193/1896: Converting Node Type Shape\n",
      "194/1896: Converting Node Type Gather\n",
      "195/1896: Converting Node Type Shape\n",
      "196/1896: Converting Node Type Gather\n",
      "197/1896: Converting Node Type Unsqueeze\n",
      "198/1896: Converting Node Type Unsqueeze\n",
      "199/1896: Converting Node Type Concat\n",
      "200/1896: Converting Node Type Reshape\n",
      "201/1896: Converting Node Type Shape\n",
      "202/1896: Converting Node Type Gather\n",
      "203/1896: Converting Node Type Shape\n",
      "204/1896: Converting Node Type Gather\n",
      "205/1896: Converting Node Type Unsqueeze\n",
      "206/1896: Converting Node Type Unsqueeze\n",
      "207/1896: Converting Node Type Concat\n",
      "208/1896: Converting Node Type Reshape\n",
      "209/1896: Converting Node Type Transpose\n",
      "210/1896: Converting Node Type Transpose\n",
      "211/1896: Converting Node Type MatMul\n",
      "212/1896: Converting Node Type Div\n",
      "213/1896: Converting Node Type Add\n",
      "214/1896: Converting Node Type Softmax\n",
      "215/1896: Converting Node Type MatMul\n",
      "216/1896: Converting Node Type Transpose\n",
      "217/1896: Converting Node Type Shape\n",
      "218/1896: Converting Node Type Gather\n",
      "219/1896: Converting Node Type Shape\n",
      "220/1896: Converting Node Type Gather\n",
      "221/1896: Converting Node Type Unsqueeze\n",
      "222/1896: Converting Node Type Unsqueeze\n",
      "223/1896: Converting Node Type Concat\n",
      "224/1896: Converting Node Type Reshape\n",
      "225/1896: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "226/1896: Converting Node Type Add\n",
      "227/1896: Converting Node Type Add\n",
      "228/1896: Converting Node Type ReduceMean\n",
      "229/1896: Converting Node Type Sub\n",
      "230/1896: Converting Node Type Pow\n",
      "231/1896: Converting Node Type ReduceMean\n",
      "232/1896: Converting Node Type Add\n",
      "233/1896: Converting Node Type Sqrt\n",
      "234/1896: Converting Node Type Div\n",
      "235/1896: Converting Node Type Mul\n",
      "236/1896: Converting Node Type Add\n",
      "237/1896: Converting Node Type MatMul\n",
      "238/1896: Converting Node Type Add\n",
      "239/1896: Converting Node Type Mul\n",
      "240/1896: Converting Node Type Div\n",
      "241/1896: Converting Node Type Erf\n",
      "242/1896: Converting Node Type Add\n",
      "243/1896: Converting Node Type Mul\n",
      "244/1896: Converting Node Type MatMul\n",
      "245/1896: Converting Node Type Add\n",
      "246/1896: Converting Node Type Add\n",
      "247/1896: Converting Node Type ReduceMean\n",
      "248/1896: Converting Node Type Sub\n",
      "249/1896: Converting Node Type Pow\n",
      "250/1896: Converting Node Type ReduceMean\n",
      "251/1896: Converting Node Type Add\n",
      "252/1896: Converting Node Type Sqrt\n",
      "253/1896: Converting Node Type Div\n",
      "254/1896: Converting Node Type Mul\n",
      "255/1896: Converting Node Type Add\n",
      "256/1896: Converting Node Type MatMul\n",
      "257/1896: Converting Node Type Add\n",
      "258/1896: Converting Node Type MatMul\n",
      "259/1896: Converting Node Type Add\n",
      "260/1896: Converting Node Type MatMul\n",
      "261/1896: Converting Node Type Add\n",
      "262/1896: Converting Node Type Shape\n",
      "263/1896: Converting Node Type Gather\n",
      "264/1896: Converting Node Type Shape\n",
      "265/1896: Converting Node Type Gather\n",
      "266/1896: Converting Node Type Unsqueeze\n",
      "267/1896: Converting Node Type Unsqueeze\n",
      "268/1896: Converting Node Type Concat\n",
      "269/1896: Converting Node Type Reshape\n",
      "270/1896: Converting Node Type Transpose\n",
      "271/1896: Converting Node Type Shape\n",
      "272/1896: Converting Node Type Gather\n",
      "273/1896: Converting Node Type Shape\n",
      "274/1896: Converting Node Type Gather\n",
      "275/1896: Converting Node Type Unsqueeze\n",
      "276/1896: Converting Node Type Unsqueeze\n",
      "277/1896: Converting Node Type Concat\n",
      "278/1896: Converting Node Type Reshape\n",
      "279/1896: Converting Node Type Shape\n",
      "280/1896: Converting Node Type Gather\n",
      "281/1896: Converting Node Type Shape\n",
      "282/1896: Converting Node Type Gather\n",
      "283/1896: Converting Node Type Unsqueeze\n",
      "284/1896: Converting Node Type Unsqueeze\n",
      "285/1896: Converting Node Type Concat\n",
      "286/1896: Converting Node Type Reshape\n",
      "287/1896: Converting Node Type Transpose\n",
      "288/1896: Converting Node Type Transpose\n",
      "289/1896: Converting Node Type MatMul\n",
      "290/1896: Converting Node Type Div\n",
      "291/1896: Converting Node Type Add\n",
      "292/1896: Converting Node Type Softmax\n",
      "293/1896: Converting Node Type MatMul\n",
      "294/1896: Converting Node Type Transpose\n",
      "295/1896: Converting Node Type Shape\n",
      "296/1896: Converting Node Type Gather\n",
      "297/1896: Converting Node Type Shape\n",
      "298/1896: Converting Node Type Gather\n",
      "299/1896: Converting Node Type Unsqueeze\n",
      "300/1896: Converting Node Type Unsqueeze\n",
      "301/1896: Converting Node Type Concat\n",
      "302/1896: Converting Node Type Reshape\n",
      "303/1896: Converting Node Type MatMul\n",
      "304/1896: Converting Node Type Add\n",
      "305/1896: Converting Node Type Add\n",
      "306/1896: Converting Node Type ReduceMean\n",
      "307/1896: Converting Node Type Sub\n",
      "308/1896: Converting Node Type Pow\n",
      "309/1896: Converting Node Type ReduceMean\n",
      "310/1896: Converting Node Type Add\n",
      "311/1896: Converting Node Type Sqrt\n",
      "312/1896: Converting Node Type Div\n",
      "313/1896: Converting Node Type Mul\n",
      "314/1896: Converting Node Type Add\n",
      "315/1896: Converting Node Type MatMul\n",
      "316/1896: Converting Node Type Add\n",
      "317/1896: Converting Node Type Mul\n",
      "318/1896: Converting Node Type Div\n",
      "319/1896: Converting Node Type Erf\n",
      "320/1896: Converting Node Type Add\n",
      "321/1896: Converting Node Type Mul\n",
      "322/1896: Converting Node Type MatMul\n",
      "323/1896: Converting Node Type Add\n",
      "324/1896: Converting Node Type Add\n",
      "325/1896: Converting Node Type ReduceMean\n",
      "326/1896: Converting Node Type Sub\n",
      "327/1896: Converting Node Type Pow\n",
      "328/1896: Converting Node Type ReduceMean\n",
      "329/1896: Converting Node Type Add\n",
      "330/1896: Converting Node Type Sqrt\n",
      "331/1896: Converting Node Type Div\n",
      "332/1896: Converting Node Type Mul\n",
      "333/1896: Converting Node Type Add\n",
      "334/1896: Converting Node Type MatMul\n",
      "335/1896: Converting Node Type Add\n",
      "336/1896: Converting Node Type MatMul\n",
      "337/1896: Converting Node Type Add\n",
      "338/1896: Converting Node Type MatMul\n",
      "339/1896: Converting Node Type Add\n",
      "340/1896: Converting Node Type Shape\n",
      "341/1896: Converting Node Type Gather\n",
      "342/1896: Converting Node Type Shape\n",
      "343/1896: Converting Node Type Gather\n",
      "344/1896: Converting Node Type Unsqueeze\n",
      "345/1896: Converting Node Type Unsqueeze\n",
      "346/1896: Converting Node Type Concat\n",
      "347/1896: Converting Node Type Reshape\n",
      "348/1896: Converting Node Type Transpose\n",
      "349/1896: Converting Node Type Shape\n",
      "350/1896: Converting Node Type Gather\n",
      "351/1896: Converting Node Type Shape\n",
      "352/1896: Converting Node Type Gather\n",
      "353/1896: Converting Node Type Unsqueeze\n",
      "354/1896: Converting Node Type Unsqueeze\n",
      "355/1896: Converting Node Type Concat\n",
      "356/1896: Converting Node Type Reshape\n",
      "357/1896: Converting Node Type Shape\n",
      "358/1896: Converting Node Type Gather\n",
      "359/1896: Converting Node Type Shape\n",
      "360/1896: Converting Node Type Gather\n",
      "361/1896: Converting Node Type Unsqueeze\n",
      "362/1896: Converting Node Type Unsqueeze\n",
      "363/1896: Converting Node Type Concat\n",
      "364/1896: Converting Node Type Reshape\n",
      "365/1896: Converting Node Type Transpose\n",
      "366/1896: Converting Node Type Transpose\n",
      "367/1896: Converting Node Type MatMul\n",
      "368/1896: Converting Node Type Div\n",
      "369/1896: Converting Node Type Add\n",
      "370/1896: Converting Node Type Softmax\n",
      "371/1896: Converting Node Type MatMul\n",
      "372/1896: Converting Node Type Transpose\n",
      "373/1896: Converting Node Type Shape\n",
      "374/1896: Converting Node Type Gather\n",
      "375/1896: Converting Node Type Shape\n",
      "376/1896: Converting Node Type Gather\n",
      "377/1896: Converting Node Type Unsqueeze\n",
      "378/1896: Converting Node Type Unsqueeze\n",
      "379/1896: Converting Node Type Concat\n",
      "380/1896: Converting Node Type Reshape\n",
      "381/1896: Converting Node Type MatMul\n",
      "382/1896: Converting Node Type Add\n",
      "383/1896: Converting Node Type Add\n",
      "384/1896: Converting Node Type ReduceMean\n",
      "385/1896: Converting Node Type Sub\n",
      "386/1896: Converting Node Type Pow\n",
      "387/1896: Converting Node Type ReduceMean\n",
      "388/1896: Converting Node Type Add\n",
      "389/1896: Converting Node Type Sqrt\n",
      "390/1896: Converting Node Type Div\n",
      "391/1896: Converting Node Type Mul\n",
      "392/1896: Converting Node Type Add\n",
      "393/1896: Converting Node Type MatMul\n",
      "394/1896: Converting Node Type Add\n",
      "395/1896: Converting Node Type Mul\n",
      "396/1896: Converting Node Type Div\n",
      "397/1896: Converting Node Type Erf\n",
      "398/1896: Converting Node Type Add\n",
      "399/1896: Converting Node Type Mul\n",
      "400/1896: Converting Node Type MatMul\n",
      "401/1896: Converting Node Type Add\n",
      "402/1896: Converting Node Type Add\n",
      "403/1896: Converting Node Type ReduceMean\n",
      "404/1896: Converting Node Type Sub\n",
      "405/1896: Converting Node Type Pow\n",
      "406/1896: Converting Node Type ReduceMean\n",
      "407/1896: Converting Node Type Add\n",
      "408/1896: Converting Node Type Sqrt\n",
      "409/1896: Converting Node Type Div\n",
      "410/1896: Converting Node Type Mul\n",
      "411/1896: Converting Node Type Add\n",
      "412/1896: Converting Node Type MatMul\n",
      "413/1896: Converting Node Type Add\n",
      "414/1896: Converting Node Type MatMul\n",
      "415/1896: Converting Node Type Add\n",
      "416/1896: Converting Node Type MatMul\n",
      "417/1896: Converting Node Type Add\n",
      "418/1896: Converting Node Type Shape\n",
      "419/1896: Converting Node Type Gather\n",
      "420/1896: Converting Node Type Shape\n",
      "421/1896: Converting Node Type Gather\n",
      "422/1896: Converting Node Type Unsqueeze\n",
      "423/1896: Converting Node Type Unsqueeze\n",
      "424/1896: Converting Node Type Concat\n",
      "425/1896: Converting Node Type Reshape\n",
      "426/1896: Converting Node Type Transpose\n",
      "427/1896: Converting Node Type Shape\n",
      "428/1896: Converting Node Type Gather\n",
      "429/1896: Converting Node Type Shape\n",
      "430/1896: Converting Node Type Gather\n",
      "431/1896: Converting Node Type Unsqueeze\n",
      "432/1896: Converting Node Type Unsqueeze\n",
      "433/1896: Converting Node Type Concat\n",
      "434/1896: Converting Node Type Reshape\n",
      "435/1896: Converting Node Type Shape\n",
      "436/1896: Converting Node Type Gather\n",
      "437/1896: Converting Node Type Shape\n",
      "438/1896: Converting Node Type Gather\n",
      "439/1896: Converting Node Type Unsqueeze\n",
      "440/1896: Converting Node Type Unsqueeze\n",
      "441/1896: Converting Node Type Concat\n",
      "442/1896: Converting Node Type Reshape\n",
      "443/1896: Converting Node Type Transpose\n",
      "444/1896: Converting Node Type Transpose\n",
      "445/1896: Converting Node Type MatMul\n",
      "446/1896: Converting Node Type Div\n",
      "447/1896: Converting Node Type Add\n",
      "448/1896: Converting Node Type Softmax\n",
      "449/1896: Converting Node Type MatMul\n",
      "450/1896: Converting Node Type Transpose\n",
      "451/1896: Converting Node Type Shape\n",
      "452/1896: Converting Node Type Gather\n",
      "453/1896: Converting Node Type Shape\n",
      "454/1896: Converting Node Type Gather\n",
      "455/1896: Converting Node Type Unsqueeze\n",
      "456/1896: Converting Node Type Unsqueeze\n",
      "457/1896: Converting Node Type Concat\n",
      "458/1896: Converting Node Type Reshape\n",
      "459/1896: Converting Node Type MatMul\n",
      "460/1896: Converting Node Type Add\n",
      "461/1896: Converting Node Type Add\n",
      "462/1896: Converting Node Type ReduceMean\n",
      "463/1896: Converting Node Type Sub\n",
      "464/1896: Converting Node Type Pow\n",
      "465/1896: Converting Node Type ReduceMean\n",
      "466/1896: Converting Node Type Add\n",
      "467/1896: Converting Node Type Sqrt\n",
      "468/1896: Converting Node Type Div\n",
      "469/1896: Converting Node Type Mul\n",
      "470/1896: Converting Node Type Add\n",
      "471/1896: Converting Node Type MatMul\n",
      "472/1896: Converting Node Type Add\n",
      "473/1896: Converting Node Type Mul\n",
      "474/1896: Converting Node Type Div\n",
      "475/1896: Converting Node Type Erf\n",
      "476/1896: Converting Node Type Add\n",
      "477/1896: Converting Node Type Mul\n",
      "478/1896: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "479/1896: Converting Node Type Add\n",
      "480/1896: Converting Node Type Add\n",
      "481/1896: Converting Node Type ReduceMean\n",
      "482/1896: Converting Node Type Sub\n",
      "483/1896: Converting Node Type Pow\n",
      "484/1896: Converting Node Type ReduceMean\n",
      "485/1896: Converting Node Type Add\n",
      "486/1896: Converting Node Type Sqrt\n",
      "487/1896: Converting Node Type Div\n",
      "488/1896: Converting Node Type Mul\n",
      "489/1896: Converting Node Type Add\n",
      "490/1896: Converting Node Type MatMul\n",
      "491/1896: Converting Node Type Add\n",
      "492/1896: Converting Node Type MatMul\n",
      "493/1896: Converting Node Type Add\n",
      "494/1896: Converting Node Type MatMul\n",
      "495/1896: Converting Node Type Add\n",
      "496/1896: Converting Node Type Shape\n",
      "497/1896: Converting Node Type Gather\n",
      "498/1896: Converting Node Type Shape\n",
      "499/1896: Converting Node Type Gather\n",
      "500/1896: Converting Node Type Unsqueeze\n",
      "501/1896: Converting Node Type Unsqueeze\n",
      "502/1896: Converting Node Type Concat\n",
      "503/1896: Converting Node Type Reshape\n",
      "504/1896: Converting Node Type Transpose\n",
      "505/1896: Converting Node Type Shape\n",
      "506/1896: Converting Node Type Gather\n",
      "507/1896: Converting Node Type Shape\n",
      "508/1896: Converting Node Type Gather\n",
      "509/1896: Converting Node Type Unsqueeze\n",
      "510/1896: Converting Node Type Unsqueeze\n",
      "511/1896: Converting Node Type Concat\n",
      "512/1896: Converting Node Type Reshape\n",
      "513/1896: Converting Node Type Shape\n",
      "514/1896: Converting Node Type Gather\n",
      "515/1896: Converting Node Type Shape\n",
      "516/1896: Converting Node Type Gather\n",
      "517/1896: Converting Node Type Unsqueeze\n",
      "518/1896: Converting Node Type Unsqueeze\n",
      "519/1896: Converting Node Type Concat\n",
      "520/1896: Converting Node Type Reshape\n",
      "521/1896: Converting Node Type Transpose\n",
      "522/1896: Converting Node Type Transpose\n",
      "523/1896: Converting Node Type MatMul\n",
      "524/1896: Converting Node Type Div\n",
      "525/1896: Converting Node Type Add\n",
      "526/1896: Converting Node Type Softmax\n",
      "527/1896: Converting Node Type MatMul\n",
      "528/1896: Converting Node Type Transpose\n",
      "529/1896: Converting Node Type Shape\n",
      "530/1896: Converting Node Type Gather\n",
      "531/1896: Converting Node Type Shape\n",
      "532/1896: Converting Node Type Gather\n",
      "533/1896: Converting Node Type Unsqueeze\n",
      "534/1896: Converting Node Type Unsqueeze\n",
      "535/1896: Converting Node Type Concat\n",
      "536/1896: Converting Node Type Reshape\n",
      "537/1896: Converting Node Type MatMul\n",
      "538/1896: Converting Node Type Add\n",
      "539/1896: Converting Node Type Add\n",
      "540/1896: Converting Node Type ReduceMean\n",
      "541/1896: Converting Node Type Sub\n",
      "542/1896: Converting Node Type Pow\n",
      "543/1896: Converting Node Type ReduceMean\n",
      "544/1896: Converting Node Type Add\n",
      "545/1896: Converting Node Type Sqrt\n",
      "546/1896: Converting Node Type Div\n",
      "547/1896: Converting Node Type Mul\n",
      "548/1896: Converting Node Type Add\n",
      "549/1896: Converting Node Type MatMul\n",
      "550/1896: Converting Node Type Add\n",
      "551/1896: Converting Node Type Mul\n",
      "552/1896: Converting Node Type Div\n",
      "553/1896: Converting Node Type Erf\n",
      "554/1896: Converting Node Type Add\n",
      "555/1896: Converting Node Type Mul\n",
      "556/1896: Converting Node Type MatMul\n",
      "557/1896: Converting Node Type Add\n",
      "558/1896: Converting Node Type Add\n",
      "559/1896: Converting Node Type ReduceMean\n",
      "560/1896: Converting Node Type Sub\n",
      "561/1896: Converting Node Type Pow\n",
      "562/1896: Converting Node Type ReduceMean\n",
      "563/1896: Converting Node Type Add\n",
      "564/1896: Converting Node Type Sqrt\n",
      "565/1896: Converting Node Type Div\n",
      "566/1896: Converting Node Type Mul\n",
      "567/1896: Converting Node Type Add\n",
      "568/1896: Converting Node Type MatMul\n",
      "569/1896: Converting Node Type Add\n",
      "570/1896: Converting Node Type MatMul\n",
      "571/1896: Converting Node Type Add\n",
      "572/1896: Converting Node Type MatMul\n",
      "573/1896: Converting Node Type Add\n",
      "574/1896: Converting Node Type Shape\n",
      "575/1896: Converting Node Type Gather\n",
      "576/1896: Converting Node Type Shape\n",
      "577/1896: Converting Node Type Gather\n",
      "578/1896: Converting Node Type Unsqueeze\n",
      "579/1896: Converting Node Type Unsqueeze\n",
      "580/1896: Converting Node Type Concat\n",
      "581/1896: Converting Node Type Reshape\n",
      "582/1896: Converting Node Type Transpose\n",
      "583/1896: Converting Node Type Shape\n",
      "584/1896: Converting Node Type Gather\n",
      "585/1896: Converting Node Type Shape\n",
      "586/1896: Converting Node Type Gather\n",
      "587/1896: Converting Node Type Unsqueeze\n",
      "588/1896: Converting Node Type Unsqueeze\n",
      "589/1896: Converting Node Type Concat\n",
      "590/1896: Converting Node Type Reshape\n",
      "591/1896: Converting Node Type Shape\n",
      "592/1896: Converting Node Type Gather\n",
      "593/1896: Converting Node Type Shape\n",
      "594/1896: Converting Node Type Gather\n",
      "595/1896: Converting Node Type Unsqueeze\n",
      "596/1896: Converting Node Type Unsqueeze\n",
      "597/1896: Converting Node Type Concat\n",
      "598/1896: Converting Node Type Reshape\n",
      "599/1896: Converting Node Type Transpose\n",
      "600/1896: Converting Node Type Transpose\n",
      "601/1896: Converting Node Type MatMul\n",
      "602/1896: Converting Node Type Div\n",
      "603/1896: Converting Node Type Add\n",
      "604/1896: Converting Node Type Softmax\n",
      "605/1896: Converting Node Type MatMul\n",
      "606/1896: Converting Node Type Transpose\n",
      "607/1896: Converting Node Type Shape\n",
      "608/1896: Converting Node Type Gather\n",
      "609/1896: Converting Node Type Shape\n",
      "610/1896: Converting Node Type Gather\n",
      "611/1896: Converting Node Type Unsqueeze\n",
      "612/1896: Converting Node Type Unsqueeze\n",
      "613/1896: Converting Node Type Concat\n",
      "614/1896: Converting Node Type Reshape\n",
      "615/1896: Converting Node Type MatMul\n",
      "616/1896: Converting Node Type Add\n",
      "617/1896: Converting Node Type Add\n",
      "618/1896: Converting Node Type ReduceMean\n",
      "619/1896: Converting Node Type Sub\n",
      "620/1896: Converting Node Type Pow\n",
      "621/1896: Converting Node Type ReduceMean\n",
      "622/1896: Converting Node Type Add\n",
      "623/1896: Converting Node Type Sqrt\n",
      "624/1896: Converting Node Type Div\n",
      "625/1896: Converting Node Type Mul\n",
      "626/1896: Converting Node Type Add\n",
      "627/1896: Converting Node Type MatMul\n",
      "628/1896: Converting Node Type Add\n",
      "629/1896: Converting Node Type Mul\n",
      "630/1896: Converting Node Type Div\n",
      "631/1896: Converting Node Type Erf\n",
      "632/1896: Converting Node Type Add\n",
      "633/1896: Converting Node Type Mul\n",
      "634/1896: Converting Node Type MatMul\n",
      "635/1896: Converting Node Type Add\n",
      "636/1896: Converting Node Type Add\n",
      "637/1896: Converting Node Type ReduceMean\n",
      "638/1896: Converting Node Type Sub\n",
      "639/1896: Converting Node Type Pow\n",
      "640/1896: Converting Node Type ReduceMean\n",
      "641/1896: Converting Node Type Add\n",
      "642/1896: Converting Node Type Sqrt\n",
      "643/1896: Converting Node Type Div\n",
      "644/1896: Converting Node Type Mul\n",
      "645/1896: Converting Node Type Add\n",
      "646/1896: Converting Node Type MatMul\n",
      "647/1896: Converting Node Type Add\n",
      "648/1896: Converting Node Type MatMul\n",
      "649/1896: Converting Node Type Add\n",
      "650/1896: Converting Node Type MatMul\n",
      "651/1896: Converting Node Type Add\n",
      "652/1896: Converting Node Type Shape\n",
      "653/1896: Converting Node Type Gather\n",
      "654/1896: Converting Node Type Shape\n",
      "655/1896: Converting Node Type Gather\n",
      "656/1896: Converting Node Type Unsqueeze\n",
      "657/1896: Converting Node Type Unsqueeze\n",
      "658/1896: Converting Node Type Concat\n",
      "659/1896: Converting Node Type Reshape\n",
      "660/1896: Converting Node Type Transpose\n",
      "661/1896: Converting Node Type Shape\n",
      "662/1896: Converting Node Type Gather\n",
      "663/1896: Converting Node Type Shape\n",
      "664/1896: Converting Node Type Gather\n",
      "665/1896: Converting Node Type Unsqueeze\n",
      "666/1896: Converting Node Type Unsqueeze\n",
      "667/1896: Converting Node Type Concat\n",
      "668/1896: Converting Node Type Reshape\n",
      "669/1896: Converting Node Type Shape\n",
      "670/1896: Converting Node Type Gather\n",
      "671/1896: Converting Node Type Shape\n",
      "672/1896: Converting Node Type Gather\n",
      "673/1896: Converting Node Type Unsqueeze\n",
      "674/1896: Converting Node Type Unsqueeze\n",
      "675/1896: Converting Node Type Concat\n",
      "676/1896: Converting Node Type Reshape\n",
      "677/1896: Converting Node Type Transpose\n",
      "678/1896: Converting Node Type Transpose\n",
      "679/1896: Converting Node Type MatMul\n",
      "680/1896: Converting Node Type Div\n",
      "681/1896: Converting Node Type Add\n",
      "682/1896: Converting Node Type Softmax\n",
      "683/1896: Converting Node Type MatMul\n",
      "684/1896: Converting Node Type Transpose\n",
      "685/1896: Converting Node Type Shape\n",
      "686/1896: Converting Node Type Gather\n",
      "687/1896: Converting Node Type Shape\n",
      "688/1896: Converting Node Type Gather\n",
      "689/1896: Converting Node Type Unsqueeze\n",
      "690/1896: Converting Node Type Unsqueeze\n",
      "691/1896: Converting Node Type Concat\n",
      "692/1896: Converting Node Type Reshape\n",
      "693/1896: Converting Node Type MatMul\n",
      "694/1896: Converting Node Type Add\n",
      "695/1896: Converting Node Type Add\n",
      "696/1896: Converting Node Type ReduceMean\n",
      "697/1896: Converting Node Type Sub\n",
      "698/1896: Converting Node Type Pow\n",
      "699/1896: Converting Node Type ReduceMean\n",
      "700/1896: Converting Node Type Add\n",
      "701/1896: Converting Node Type Sqrt\n",
      "702/1896: Converting Node Type Div\n",
      "703/1896: Converting Node Type Mul\n",
      "704/1896: Converting Node Type Add\n",
      "705/1896: Converting Node Type MatMul\n",
      "706/1896: Converting Node Type Add\n",
      "707/1896: Converting Node Type Mul\n",
      "708/1896: Converting Node Type Div\n",
      "709/1896: Converting Node Type Erf\n",
      "710/1896: Converting Node Type Add\n",
      "711/1896: Converting Node Type Mul\n",
      "712/1896: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "713/1896: Converting Node Type Add\n",
      "714/1896: Converting Node Type Add\n",
      "715/1896: Converting Node Type ReduceMean\n",
      "716/1896: Converting Node Type Sub\n",
      "717/1896: Converting Node Type Pow\n",
      "718/1896: Converting Node Type ReduceMean\n",
      "719/1896: Converting Node Type Add\n",
      "720/1896: Converting Node Type Sqrt\n",
      "721/1896: Converting Node Type Div\n",
      "722/1896: Converting Node Type Mul\n",
      "723/1896: Converting Node Type Add\n",
      "724/1896: Converting Node Type MatMul\n",
      "725/1896: Converting Node Type Add\n",
      "726/1896: Converting Node Type MatMul\n",
      "727/1896: Converting Node Type Add\n",
      "728/1896: Converting Node Type MatMul\n",
      "729/1896: Converting Node Type Add\n",
      "730/1896: Converting Node Type Shape\n",
      "731/1896: Converting Node Type Gather\n",
      "732/1896: Converting Node Type Shape\n",
      "733/1896: Converting Node Type Gather\n",
      "734/1896: Converting Node Type Unsqueeze\n",
      "735/1896: Converting Node Type Unsqueeze\n",
      "736/1896: Converting Node Type Concat\n",
      "737/1896: Converting Node Type Reshape\n",
      "738/1896: Converting Node Type Transpose\n",
      "739/1896: Converting Node Type Shape\n",
      "740/1896: Converting Node Type Gather\n",
      "741/1896: Converting Node Type Shape\n",
      "742/1896: Converting Node Type Gather\n",
      "743/1896: Converting Node Type Unsqueeze\n",
      "744/1896: Converting Node Type Unsqueeze\n",
      "745/1896: Converting Node Type Concat\n",
      "746/1896: Converting Node Type Reshape\n",
      "747/1896: Converting Node Type Shape\n",
      "748/1896: Converting Node Type Gather\n",
      "749/1896: Converting Node Type Shape\n",
      "750/1896: Converting Node Type Gather\n",
      "751/1896: Converting Node Type Unsqueeze\n",
      "752/1896: Converting Node Type Unsqueeze\n",
      "753/1896: Converting Node Type Concat\n",
      "754/1896: Converting Node Type Reshape\n",
      "755/1896: Converting Node Type Transpose\n",
      "756/1896: Converting Node Type Transpose\n",
      "757/1896: Converting Node Type MatMul\n",
      "758/1896: Converting Node Type Div\n",
      "759/1896: Converting Node Type Add\n",
      "760/1896: Converting Node Type Softmax\n",
      "761/1896: Converting Node Type MatMul\n",
      "762/1896: Converting Node Type Transpose\n",
      "763/1896: Converting Node Type Shape\n",
      "764/1896: Converting Node Type Gather\n",
      "765/1896: Converting Node Type Shape\n",
      "766/1896: Converting Node Type Gather\n",
      "767/1896: Converting Node Type Unsqueeze\n",
      "768/1896: Converting Node Type Unsqueeze\n",
      "769/1896: Converting Node Type Concat\n",
      "770/1896: Converting Node Type Reshape\n",
      "771/1896: Converting Node Type MatMul\n",
      "772/1896: Converting Node Type Add\n",
      "773/1896: Converting Node Type Add\n",
      "774/1896: Converting Node Type ReduceMean\n",
      "775/1896: Converting Node Type Sub\n",
      "776/1896: Converting Node Type Pow\n",
      "777/1896: Converting Node Type ReduceMean\n",
      "778/1896: Converting Node Type Add\n",
      "779/1896: Converting Node Type Sqrt\n",
      "780/1896: Converting Node Type Div\n",
      "781/1896: Converting Node Type Mul\n",
      "782/1896: Converting Node Type Add\n",
      "783/1896: Converting Node Type MatMul\n",
      "784/1896: Converting Node Type Add\n",
      "785/1896: Converting Node Type Mul\n",
      "786/1896: Converting Node Type Div\n",
      "787/1896: Converting Node Type Erf\n",
      "788/1896: Converting Node Type Add\n",
      "789/1896: Converting Node Type Mul\n",
      "790/1896: Converting Node Type MatMul\n",
      "791/1896: Converting Node Type Add\n",
      "792/1896: Converting Node Type Add\n",
      "793/1896: Converting Node Type ReduceMean\n",
      "794/1896: Converting Node Type Sub\n",
      "795/1896: Converting Node Type Pow\n",
      "796/1896: Converting Node Type ReduceMean\n",
      "797/1896: Converting Node Type Add\n",
      "798/1896: Converting Node Type Sqrt\n",
      "799/1896: Converting Node Type Div\n",
      "800/1896: Converting Node Type Mul\n",
      "801/1896: Converting Node Type Add\n",
      "802/1896: Converting Node Type MatMul\n",
      "803/1896: Converting Node Type Add\n",
      "804/1896: Converting Node Type MatMul\n",
      "805/1896: Converting Node Type Add\n",
      "806/1896: Converting Node Type MatMul\n",
      "807/1896: Converting Node Type Add\n",
      "808/1896: Converting Node Type Shape\n",
      "809/1896: Converting Node Type Gather\n",
      "810/1896: Converting Node Type Shape\n",
      "811/1896: Converting Node Type Gather\n",
      "812/1896: Converting Node Type Unsqueeze\n",
      "813/1896: Converting Node Type Unsqueeze\n",
      "814/1896: Converting Node Type Concat\n",
      "815/1896: Converting Node Type Reshape\n",
      "816/1896: Converting Node Type Transpose\n",
      "817/1896: Converting Node Type Shape\n",
      "818/1896: Converting Node Type Gather\n",
      "819/1896: Converting Node Type Shape\n",
      "820/1896: Converting Node Type Gather\n",
      "821/1896: Converting Node Type Unsqueeze\n",
      "822/1896: Converting Node Type Unsqueeze\n",
      "823/1896: Converting Node Type Concat\n",
      "824/1896: Converting Node Type Reshape\n",
      "825/1896: Converting Node Type Shape\n",
      "826/1896: Converting Node Type Gather\n",
      "827/1896: Converting Node Type Shape\n",
      "828/1896: Converting Node Type Gather\n",
      "829/1896: Converting Node Type Unsqueeze\n",
      "830/1896: Converting Node Type Unsqueeze\n",
      "831/1896: Converting Node Type Concat\n",
      "832/1896: Converting Node Type Reshape\n",
      "833/1896: Converting Node Type Transpose\n",
      "834/1896: Converting Node Type Transpose\n",
      "835/1896: Converting Node Type MatMul\n",
      "836/1896: Converting Node Type Div\n",
      "837/1896: Converting Node Type Add\n",
      "838/1896: Converting Node Type Softmax\n",
      "839/1896: Converting Node Type MatMul\n",
      "840/1896: Converting Node Type Transpose\n",
      "841/1896: Converting Node Type Shape\n",
      "842/1896: Converting Node Type Gather\n",
      "843/1896: Converting Node Type Shape\n",
      "844/1896: Converting Node Type Gather\n",
      "845/1896: Converting Node Type Unsqueeze\n",
      "846/1896: Converting Node Type Unsqueeze\n",
      "847/1896: Converting Node Type Concat\n",
      "848/1896: Converting Node Type Reshape\n",
      "849/1896: Converting Node Type MatMul\n",
      "850/1896: Converting Node Type Add\n",
      "851/1896: Converting Node Type Add\n",
      "852/1896: Converting Node Type ReduceMean\n",
      "853/1896: Converting Node Type Sub\n",
      "854/1896: Converting Node Type Pow\n",
      "855/1896: Converting Node Type ReduceMean\n",
      "856/1896: Converting Node Type Add\n",
      "857/1896: Converting Node Type Sqrt\n",
      "858/1896: Converting Node Type Div\n",
      "859/1896: Converting Node Type Mul\n",
      "860/1896: Converting Node Type Add\n",
      "861/1896: Converting Node Type MatMul\n",
      "862/1896: Converting Node Type Add\n",
      "863/1896: Converting Node Type Mul\n",
      "864/1896: Converting Node Type Div\n",
      "865/1896: Converting Node Type Erf\n",
      "866/1896: Converting Node Type Add\n",
      "867/1896: Converting Node Type Mul\n",
      "868/1896: Converting Node Type MatMul\n",
      "869/1896: Converting Node Type Add\n",
      "870/1896: Converting Node Type Add\n",
      "871/1896: Converting Node Type ReduceMean\n",
      "872/1896: Converting Node Type Sub\n",
      "873/1896: Converting Node Type Pow\n",
      "874/1896: Converting Node Type ReduceMean\n",
      "875/1896: Converting Node Type Add\n",
      "876/1896: Converting Node Type Sqrt\n",
      "877/1896: Converting Node Type Div\n",
      "878/1896: Converting Node Type Mul\n",
      "879/1896: Converting Node Type Add\n",
      "880/1896: Converting Node Type MatMul\n",
      "881/1896: Converting Node Type Add\n",
      "882/1896: Converting Node Type MatMul\n",
      "883/1896: Converting Node Type Add\n",
      "884/1896: Converting Node Type MatMul\n",
      "885/1896: Converting Node Type Add\n",
      "886/1896: Converting Node Type Shape\n",
      "887/1896: Converting Node Type Gather\n",
      "888/1896: Converting Node Type Shape\n",
      "889/1896: Converting Node Type Gather\n",
      "890/1896: Converting Node Type Unsqueeze\n",
      "891/1896: Converting Node Type Unsqueeze\n",
      "892/1896: Converting Node Type Concat\n",
      "893/1896: Converting Node Type Reshape\n",
      "894/1896: Converting Node Type Transpose\n",
      "895/1896: Converting Node Type Shape\n",
      "896/1896: Converting Node Type Gather\n",
      "897/1896: Converting Node Type Shape\n",
      "898/1896: Converting Node Type Gather\n",
      "899/1896: Converting Node Type Unsqueeze\n",
      "900/1896: Converting Node Type Unsqueeze\n",
      "901/1896: Converting Node Type Concat\n",
      "902/1896: Converting Node Type Reshape\n",
      "903/1896: Converting Node Type Shape\n",
      "904/1896: Converting Node Type Gather\n",
      "905/1896: Converting Node Type Shape\n",
      "906/1896: Converting Node Type Gather\n",
      "907/1896: Converting Node Type Unsqueeze\n",
      "908/1896: Converting Node Type Unsqueeze\n",
      "909/1896: Converting Node Type Concat\n",
      "910/1896: Converting Node Type Reshape\n",
      "911/1896: Converting Node Type Transpose\n",
      "912/1896: Converting Node Type Transpose\n",
      "913/1896: Converting Node Type MatMul\n",
      "914/1896: Converting Node Type Div\n",
      "915/1896: Converting Node Type Add\n",
      "916/1896: Converting Node Type Softmax\n",
      "917/1896: Converting Node Type MatMul\n",
      "918/1896: Converting Node Type Transpose\n",
      "919/1896: Converting Node Type Shape\n",
      "920/1896: Converting Node Type Gather\n",
      "921/1896: Converting Node Type Shape\n",
      "922/1896: Converting Node Type Gather\n",
      "923/1896: Converting Node Type Unsqueeze\n",
      "924/1896: Converting Node Type Unsqueeze\n",
      "925/1896: Converting Node Type Concat\n",
      "926/1896: Converting Node Type Reshape\n",
      "927/1896: Converting Node Type MatMul\n",
      "928/1896: Converting Node Type Add\n",
      "929/1896: Converting Node Type Add\n",
      "930/1896: Converting Node Type ReduceMean\n",
      "931/1896: Converting Node Type Sub\n",
      "932/1896: Converting Node Type Pow\n",
      "933/1896: Converting Node Type ReduceMean\n",
      "934/1896: Converting Node Type Add\n",
      "935/1896: Converting Node Type Sqrt\n",
      "936/1896: Converting Node Type Div\n",
      "937/1896: Converting Node Type Mul\n",
      "938/1896: Converting Node Type Add\n",
      "939/1896: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "940/1896: Converting Node Type Add\n",
      "941/1896: Converting Node Type Mul\n",
      "942/1896: Converting Node Type Div\n",
      "943/1896: Converting Node Type Erf\n",
      "944/1896: Converting Node Type Add\n",
      "945/1896: Converting Node Type Mul\n",
      "946/1896: Converting Node Type MatMul\n",
      "947/1896: Converting Node Type Add\n",
      "948/1896: Converting Node Type Add\n",
      "949/1896: Converting Node Type ReduceMean\n",
      "950/1896: Converting Node Type Sub\n",
      "951/1896: Converting Node Type Pow\n",
      "952/1896: Converting Node Type ReduceMean\n",
      "953/1896: Converting Node Type Add\n",
      "954/1896: Converting Node Type Sqrt\n",
      "955/1896: Converting Node Type Div\n",
      "956/1896: Converting Node Type Mul\n",
      "957/1896: Converting Node Type Add\n",
      "958/1896: Converting Node Type MatMul\n",
      "959/1896: Converting Node Type Add\n",
      "960/1896: Converting Node Type MatMul\n",
      "961/1896: Converting Node Type Add\n",
      "962/1896: Converting Node Type MatMul\n",
      "963/1896: Converting Node Type Add\n",
      "964/1896: Converting Node Type Shape\n",
      "965/1896: Converting Node Type Gather\n",
      "966/1896: Converting Node Type Shape\n",
      "967/1896: Converting Node Type Gather\n",
      "968/1896: Converting Node Type Unsqueeze\n",
      "969/1896: Converting Node Type Unsqueeze\n",
      "970/1896: Converting Node Type Concat\n",
      "971/1896: Converting Node Type Reshape\n",
      "972/1896: Converting Node Type Transpose\n",
      "973/1896: Converting Node Type Shape\n",
      "974/1896: Converting Node Type Gather\n",
      "975/1896: Converting Node Type Shape\n",
      "976/1896: Converting Node Type Gather\n",
      "977/1896: Converting Node Type Unsqueeze\n",
      "978/1896: Converting Node Type Unsqueeze\n",
      "979/1896: Converting Node Type Concat\n",
      "980/1896: Converting Node Type Reshape\n",
      "981/1896: Converting Node Type Shape\n",
      "982/1896: Converting Node Type Gather\n",
      "983/1896: Converting Node Type Shape\n",
      "984/1896: Converting Node Type Gather\n",
      "985/1896: Converting Node Type Unsqueeze\n",
      "986/1896: Converting Node Type Unsqueeze\n",
      "987/1896: Converting Node Type Concat\n",
      "988/1896: Converting Node Type Reshape\n",
      "989/1896: Converting Node Type Transpose\n",
      "990/1896: Converting Node Type Transpose\n",
      "991/1896: Converting Node Type MatMul\n",
      "992/1896: Converting Node Type Div\n",
      "993/1896: Converting Node Type Add\n",
      "994/1896: Converting Node Type Softmax\n",
      "995/1896: Converting Node Type MatMul\n",
      "996/1896: Converting Node Type Transpose\n",
      "997/1896: Converting Node Type Shape\n",
      "998/1896: Converting Node Type Gather\n",
      "999/1896: Converting Node Type Shape\n",
      "1000/1896: Converting Node Type Gather\n",
      "1001/1896: Converting Node Type Unsqueeze\n",
      "1002/1896: Converting Node Type Unsqueeze\n",
      "1003/1896: Converting Node Type Concat\n",
      "1004/1896: Converting Node Type Reshape\n",
      "1005/1896: Converting Node Type MatMul\n",
      "1006/1896: Converting Node Type Add\n",
      "1007/1896: Converting Node Type Add\n",
      "1008/1896: Converting Node Type ReduceMean\n",
      "1009/1896: Converting Node Type Sub\n",
      "1010/1896: Converting Node Type Pow\n",
      "1011/1896: Converting Node Type ReduceMean\n",
      "1012/1896: Converting Node Type Add\n",
      "1013/1896: Converting Node Type Sqrt\n",
      "1014/1896: Converting Node Type Div\n",
      "1015/1896: Converting Node Type Mul\n",
      "1016/1896: Converting Node Type Add\n",
      "1017/1896: Converting Node Type MatMul\n",
      "1018/1896: Converting Node Type Add\n",
      "1019/1896: Converting Node Type Mul\n",
      "1020/1896: Converting Node Type Div\n",
      "1021/1896: Converting Node Type Erf\n",
      "1022/1896: Converting Node Type Add\n",
      "1023/1896: Converting Node Type Mul\n",
      "1024/1896: Converting Node Type MatMul\n",
      "1025/1896: Converting Node Type Add\n",
      "1026/1896: Converting Node Type Add\n",
      "1027/1896: Converting Node Type ReduceMean\n",
      "1028/1896: Converting Node Type Sub\n",
      "1029/1896: Converting Node Type Pow\n",
      "1030/1896: Converting Node Type ReduceMean\n",
      "1031/1896: Converting Node Type Add\n",
      "1032/1896: Converting Node Type Sqrt\n",
      "1033/1896: Converting Node Type Div\n",
      "1034/1896: Converting Node Type Mul\n",
      "1035/1896: Converting Node Type Add\n",
      "1036/1896: Converting Node Type MatMul\n",
      "1037/1896: Converting Node Type Add\n",
      "1038/1896: Converting Node Type MatMul\n",
      "1039/1896: Converting Node Type Add\n",
      "1040/1896: Converting Node Type MatMul\n",
      "1041/1896: Converting Node Type Add\n",
      "1042/1896: Converting Node Type Shape\n",
      "1043/1896: Converting Node Type Gather\n",
      "1044/1896: Converting Node Type Shape\n",
      "1045/1896: Converting Node Type Gather\n",
      "1046/1896: Converting Node Type Unsqueeze\n",
      "1047/1896: Converting Node Type Unsqueeze\n",
      "1048/1896: Converting Node Type Concat\n",
      "1049/1896: Converting Node Type Reshape\n",
      "1050/1896: Converting Node Type Transpose\n",
      "1051/1896: Converting Node Type Shape\n",
      "1052/1896: Converting Node Type Gather\n",
      "1053/1896: Converting Node Type Shape\n",
      "1054/1896: Converting Node Type Gather\n",
      "1055/1896: Converting Node Type Unsqueeze\n",
      "1056/1896: Converting Node Type Unsqueeze\n",
      "1057/1896: Converting Node Type Concat\n",
      "1058/1896: Converting Node Type Reshape\n",
      "1059/1896: Converting Node Type Shape\n",
      "1060/1896: Converting Node Type Gather\n",
      "1061/1896: Converting Node Type Shape\n",
      "1062/1896: Converting Node Type Gather\n",
      "1063/1896: Converting Node Type Unsqueeze\n",
      "1064/1896: Converting Node Type Unsqueeze\n",
      "1065/1896: Converting Node Type Concat\n",
      "1066/1896: Converting Node Type Reshape\n",
      "1067/1896: Converting Node Type Transpose\n",
      "1068/1896: Converting Node Type Transpose\n",
      "1069/1896: Converting Node Type MatMul\n",
      "1070/1896: Converting Node Type Div\n",
      "1071/1896: Converting Node Type Add\n",
      "1072/1896: Converting Node Type Softmax\n",
      "1073/1896: Converting Node Type MatMul\n",
      "1074/1896: Converting Node Type Transpose\n",
      "1075/1896: Converting Node Type Shape\n",
      "1076/1896: Converting Node Type Gather\n",
      "1077/1896: Converting Node Type Shape\n",
      "1078/1896: Converting Node Type Gather\n",
      "1079/1896: Converting Node Type Unsqueeze\n",
      "1080/1896: Converting Node Type Unsqueeze\n",
      "1081/1896: Converting Node Type Concat\n",
      "1082/1896: Converting Node Type Reshape\n",
      "1083/1896: Converting Node Type MatMul\n",
      "1084/1896: Converting Node Type Add\n",
      "1085/1896: Converting Node Type Add\n",
      "1086/1896: Converting Node Type ReduceMean\n",
      "1087/1896: Converting Node Type Sub\n",
      "1088/1896: Converting Node Type Pow\n",
      "1089/1896: Converting Node Type ReduceMean\n",
      "1090/1896: Converting Node Type Add\n",
      "1091/1896: Converting Node Type Sqrt\n",
      "1092/1896: Converting Node Type Div\n",
      "1093/1896: Converting Node Type Mul\n",
      "1094/1896: Converting Node Type Add\n",
      "1095/1896: Converting Node Type MatMul\n",
      "1096/1896: Converting Node Type Add\n",
      "1097/1896: Converting Node Type Mul\n",
      "1098/1896: Converting Node Type Div\n",
      "1099/1896: Converting Node Type Erf\n",
      "1100/1896: Converting Node Type Add\n",
      "1101/1896: Converting Node Type Mul\n",
      "1102/1896: Converting Node Type MatMul\n",
      "1103/1896: Converting Node Type Add\n",
      "1104/1896: Converting Node Type Add\n",
      "1105/1896: Converting Node Type ReduceMean\n",
      "1106/1896: Converting Node Type Sub\n",
      "1107/1896: Converting Node Type Pow\n",
      "1108/1896: Converting Node Type ReduceMean\n",
      "1109/1896: Converting Node Type Add\n",
      "1110/1896: Converting Node Type Sqrt\n",
      "1111/1896: Converting Node Type Div\n",
      "1112/1896: Converting Node Type Mul\n",
      "1113/1896: Converting Node Type Add\n",
      "1114/1896: Converting Node Type MatMul\n",
      "1115/1896: Converting Node Type Add\n",
      "1116/1896: Converting Node Type MatMul\n",
      "1117/1896: Converting Node Type Add\n",
      "1118/1896: Converting Node Type MatMul\n",
      "1119/1896: Converting Node Type Add\n",
      "1120/1896: Converting Node Type Shape\n",
      "1121/1896: Converting Node Type Gather\n",
      "1122/1896: Converting Node Type Shape\n",
      "1123/1896: Converting Node Type Gather\n",
      "1124/1896: Converting Node Type Unsqueeze\n",
      "1125/1896: Converting Node Type Unsqueeze\n",
      "1126/1896: Converting Node Type Concat\n",
      "1127/1896: Converting Node Type Reshape\n",
      "1128/1896: Converting Node Type Transpose\n",
      "1129/1896: Converting Node Type Shape\n",
      "1130/1896: Converting Node Type Gather\n",
      "1131/1896: Converting Node Type Shape\n",
      "1132/1896: Converting Node Type Gather\n",
      "1133/1896: Converting Node Type Unsqueeze\n",
      "1134/1896: Converting Node Type Unsqueeze\n",
      "1135/1896: Converting Node Type Concat\n",
      "1136/1896: Converting Node Type Reshape\n",
      "1137/1896: Converting Node Type Shape\n",
      "1138/1896: Converting Node Type Gather\n",
      "1139/1896: Converting Node Type Shape\n",
      "1140/1896: Converting Node Type Gather\n",
      "1141/1896: Converting Node Type Unsqueeze\n",
      "1142/1896: Converting Node Type Unsqueeze\n",
      "1143/1896: Converting Node Type Concat\n",
      "1144/1896: Converting Node Type Reshape\n",
      "1145/1896: Converting Node Type Transpose\n",
      "1146/1896: Converting Node Type Transpose\n",
      "1147/1896: Converting Node Type MatMul\n",
      "1148/1896: Converting Node Type Div\n",
      "1149/1896: Converting Node Type Add\n",
      "1150/1896: Converting Node Type Softmax\n",
      "1151/1896: Converting Node Type MatMul\n",
      "1152/1896: Converting Node Type Transpose\n",
      "1153/1896: Converting Node Type Shape\n",
      "1154/1896: Converting Node Type Gather\n",
      "1155/1896: Converting Node Type Shape\n",
      "1156/1896: Converting Node Type Gather\n",
      "1157/1896: Converting Node Type Unsqueeze\n",
      "1158/1896: Converting Node Type Unsqueeze\n",
      "1159/1896: Converting Node Type Concat\n",
      "1160/1896: Converting Node Type Reshape\n",
      "1161/1896: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1162/1896: Converting Node Type Add\n",
      "1163/1896: Converting Node Type Add\n",
      "1164/1896: Converting Node Type ReduceMean\n",
      "1165/1896: Converting Node Type Sub\n",
      "1166/1896: Converting Node Type Pow\n",
      "1167/1896: Converting Node Type ReduceMean\n",
      "1168/1896: Converting Node Type Add\n",
      "1169/1896: Converting Node Type Sqrt\n",
      "1170/1896: Converting Node Type Div\n",
      "1171/1896: Converting Node Type Mul\n",
      "1172/1896: Converting Node Type Add\n",
      "1173/1896: Converting Node Type MatMul\n",
      "1174/1896: Converting Node Type Add\n",
      "1175/1896: Converting Node Type Mul\n",
      "1176/1896: Converting Node Type Div\n",
      "1177/1896: Converting Node Type Erf\n",
      "1178/1896: Converting Node Type Add\n",
      "1179/1896: Converting Node Type Mul\n",
      "1180/1896: Converting Node Type MatMul\n",
      "1181/1896: Converting Node Type Add\n",
      "1182/1896: Converting Node Type Add\n",
      "1183/1896: Converting Node Type ReduceMean\n",
      "1184/1896: Converting Node Type Sub\n",
      "1185/1896: Converting Node Type Pow\n",
      "1186/1896: Converting Node Type ReduceMean\n",
      "1187/1896: Converting Node Type Add\n",
      "1188/1896: Converting Node Type Sqrt\n",
      "1189/1896: Converting Node Type Div\n",
      "1190/1896: Converting Node Type Mul\n",
      "1191/1896: Converting Node Type Add\n",
      "1192/1896: Converting Node Type MatMul\n",
      "1193/1896: Converting Node Type Add\n",
      "1194/1896: Converting Node Type MatMul\n",
      "1195/1896: Converting Node Type Add\n",
      "1196/1896: Converting Node Type MatMul\n",
      "1197/1896: Converting Node Type Add\n",
      "1198/1896: Converting Node Type Shape\n",
      "1199/1896: Converting Node Type Gather\n",
      "1200/1896: Converting Node Type Shape\n",
      "1201/1896: Converting Node Type Gather\n",
      "1202/1896: Converting Node Type Unsqueeze\n",
      "1203/1896: Converting Node Type Unsqueeze\n",
      "1204/1896: Converting Node Type Concat\n",
      "1205/1896: Converting Node Type Reshape\n",
      "1206/1896: Converting Node Type Transpose\n",
      "1207/1896: Converting Node Type Shape\n",
      "1208/1896: Converting Node Type Gather\n",
      "1209/1896: Converting Node Type Shape\n",
      "1210/1896: Converting Node Type Gather\n",
      "1211/1896: Converting Node Type Unsqueeze\n",
      "1212/1896: Converting Node Type Unsqueeze\n",
      "1213/1896: Converting Node Type Concat\n",
      "1214/1896: Converting Node Type Reshape\n",
      "1215/1896: Converting Node Type Shape\n",
      "1216/1896: Converting Node Type Gather\n",
      "1217/1896: Converting Node Type Shape\n",
      "1218/1896: Converting Node Type Gather\n",
      "1219/1896: Converting Node Type Unsqueeze\n",
      "1220/1896: Converting Node Type Unsqueeze\n",
      "1221/1896: Converting Node Type Concat\n",
      "1222/1896: Converting Node Type Reshape\n",
      "1223/1896: Converting Node Type Transpose\n",
      "1224/1896: Converting Node Type Transpose\n",
      "1225/1896: Converting Node Type MatMul\n",
      "1226/1896: Converting Node Type Div\n",
      "1227/1896: Converting Node Type Add\n",
      "1228/1896: Converting Node Type Softmax\n",
      "1229/1896: Converting Node Type MatMul\n",
      "1230/1896: Converting Node Type Transpose\n",
      "1231/1896: Converting Node Type Shape\n",
      "1232/1896: Converting Node Type Gather\n",
      "1233/1896: Converting Node Type Shape\n",
      "1234/1896: Converting Node Type Gather\n",
      "1235/1896: Converting Node Type Unsqueeze\n",
      "1236/1896: Converting Node Type Unsqueeze\n",
      "1237/1896: Converting Node Type Concat\n",
      "1238/1896: Converting Node Type Reshape\n",
      "1239/1896: Converting Node Type MatMul\n",
      "1240/1896: Converting Node Type Add\n",
      "1241/1896: Converting Node Type Add\n",
      "1242/1896: Converting Node Type ReduceMean\n",
      "1243/1896: Converting Node Type Sub\n",
      "1244/1896: Converting Node Type Pow\n",
      "1245/1896: Converting Node Type ReduceMean\n",
      "1246/1896: Converting Node Type Add\n",
      "1247/1896: Converting Node Type Sqrt\n",
      "1248/1896: Converting Node Type Div\n",
      "1249/1896: Converting Node Type Mul\n",
      "1250/1896: Converting Node Type Add\n",
      "1251/1896: Converting Node Type MatMul\n",
      "1252/1896: Converting Node Type Add\n",
      "1253/1896: Converting Node Type Mul\n",
      "1254/1896: Converting Node Type Div\n",
      "1255/1896: Converting Node Type Erf\n",
      "1256/1896: Converting Node Type Add\n",
      "1257/1896: Converting Node Type Mul\n",
      "1258/1896: Converting Node Type MatMul\n",
      "1259/1896: Converting Node Type Add\n",
      "1260/1896: Converting Node Type Add\n",
      "1261/1896: Converting Node Type ReduceMean\n",
      "1262/1896: Converting Node Type Sub\n",
      "1263/1896: Converting Node Type Pow\n",
      "1264/1896: Converting Node Type ReduceMean\n",
      "1265/1896: Converting Node Type Add\n",
      "1266/1896: Converting Node Type Sqrt\n",
      "1267/1896: Converting Node Type Div\n",
      "1268/1896: Converting Node Type Mul\n",
      "1269/1896: Converting Node Type Add\n",
      "1270/1896: Converting Node Type MatMul\n",
      "1271/1896: Converting Node Type Add\n",
      "1272/1896: Converting Node Type MatMul\n",
      "1273/1896: Converting Node Type Add\n",
      "1274/1896: Converting Node Type MatMul\n",
      "1275/1896: Converting Node Type Add\n",
      "1276/1896: Converting Node Type Shape\n",
      "1277/1896: Converting Node Type Gather\n",
      "1278/1896: Converting Node Type Shape\n",
      "1279/1896: Converting Node Type Gather\n",
      "1280/1896: Converting Node Type Unsqueeze\n",
      "1281/1896: Converting Node Type Unsqueeze\n",
      "1282/1896: Converting Node Type Concat\n",
      "1283/1896: Converting Node Type Reshape\n",
      "1284/1896: Converting Node Type Transpose\n",
      "1285/1896: Converting Node Type Shape\n",
      "1286/1896: Converting Node Type Gather\n",
      "1287/1896: Converting Node Type Shape\n",
      "1288/1896: Converting Node Type Gather\n",
      "1289/1896: Converting Node Type Unsqueeze\n",
      "1290/1896: Converting Node Type Unsqueeze\n",
      "1291/1896: Converting Node Type Concat\n",
      "1292/1896: Converting Node Type Reshape\n",
      "1293/1896: Converting Node Type Shape\n",
      "1294/1896: Converting Node Type Gather\n",
      "1295/1896: Converting Node Type Shape\n",
      "1296/1896: Converting Node Type Gather\n",
      "1297/1896: Converting Node Type Unsqueeze\n",
      "1298/1896: Converting Node Type Unsqueeze\n",
      "1299/1896: Converting Node Type Concat\n",
      "1300/1896: Converting Node Type Reshape\n",
      "1301/1896: Converting Node Type Transpose\n",
      "1302/1896: Converting Node Type Transpose\n",
      "1303/1896: Converting Node Type MatMul\n",
      "1304/1896: Converting Node Type Div\n",
      "1305/1896: Converting Node Type Add\n",
      "1306/1896: Converting Node Type Softmax\n",
      "1307/1896: Converting Node Type MatMul\n",
      "1308/1896: Converting Node Type Transpose\n",
      "1309/1896: Converting Node Type Shape\n",
      "1310/1896: Converting Node Type Gather\n",
      "1311/1896: Converting Node Type Shape\n",
      "1312/1896: Converting Node Type Gather\n",
      "1313/1896: Converting Node Type Unsqueeze\n",
      "1314/1896: Converting Node Type Unsqueeze\n",
      "1315/1896: Converting Node Type Concat\n",
      "1316/1896: Converting Node Type Reshape\n",
      "1317/1896: Converting Node Type MatMul\n",
      "1318/1896: Converting Node Type Add\n",
      "1319/1896: Converting Node Type Add\n",
      "1320/1896: Converting Node Type ReduceMean\n",
      "1321/1896: Converting Node Type Sub\n",
      "1322/1896: Converting Node Type Pow\n",
      "1323/1896: Converting Node Type ReduceMean\n",
      "1324/1896: Converting Node Type Add\n",
      "1325/1896: Converting Node Type Sqrt\n",
      "1326/1896: Converting Node Type Div\n",
      "1327/1896: Converting Node Type Mul\n",
      "1328/1896: Converting Node Type Add\n",
      "1329/1896: Converting Node Type MatMul\n",
      "1330/1896: Converting Node Type Add\n",
      "1331/1896: Converting Node Type Mul\n",
      "1332/1896: Converting Node Type Div\n",
      "1333/1896: Converting Node Type Erf\n",
      "1334/1896: Converting Node Type Add\n",
      "1335/1896: Converting Node Type Mul\n",
      "1336/1896: Converting Node Type MatMul\n",
      "1337/1896: Converting Node Type Add\n",
      "1338/1896: Converting Node Type Add\n",
      "1339/1896: Converting Node Type ReduceMean\n",
      "1340/1896: Converting Node Type Sub\n",
      "1341/1896: Converting Node Type Pow\n",
      "1342/1896: Converting Node Type ReduceMean\n",
      "1343/1896: Converting Node Type Add\n",
      "1344/1896: Converting Node Type Sqrt\n",
      "1345/1896: Converting Node Type Div\n",
      "1346/1896: Converting Node Type Mul\n",
      "1347/1896: Converting Node Type Add\n",
      "1348/1896: Converting Node Type MatMul\n",
      "1349/1896: Converting Node Type Add\n",
      "1350/1896: Converting Node Type MatMul\n",
      "1351/1896: Converting Node Type Add\n",
      "1352/1896: Converting Node Type MatMul\n",
      "1353/1896: Converting Node Type Add\n",
      "1354/1896: Converting Node Type Shape\n",
      "1355/1896: Converting Node Type Gather\n",
      "1356/1896: Converting Node Type Shape\n",
      "1357/1896: Converting Node Type Gather\n",
      "1358/1896: Converting Node Type Unsqueeze\n",
      "1359/1896: Converting Node Type Unsqueeze\n",
      "1360/1896: Converting Node Type Concat\n",
      "1361/1896: Converting Node Type Reshape\n",
      "1362/1896: Converting Node Type Transpose\n",
      "1363/1896: Converting Node Type Shape\n",
      "1364/1896: Converting Node Type Gather\n",
      "1365/1896: Converting Node Type Shape\n",
      "1366/1896: Converting Node Type Gather\n",
      "1367/1896: Converting Node Type Unsqueeze\n",
      "1368/1896: Converting Node Type Unsqueeze\n",
      "1369/1896: Converting Node Type Concat\n",
      "1370/1896: Converting Node Type Reshape\n",
      "1371/1896: Converting Node Type Shape\n",
      "1372/1896: Converting Node Type Gather\n",
      "1373/1896: Converting Node Type Shape\n",
      "1374/1896: Converting Node Type Gather\n",
      "1375/1896: Converting Node Type Unsqueeze\n",
      "1376/1896: Converting Node Type Unsqueeze\n",
      "1377/1896: Converting Node Type Concat\n",
      "1378/1896: Converting Node Type Reshape\n",
      "1379/1896: Converting Node Type Transpose\n",
      "1380/1896: Converting Node Type Transpose\n",
      "1381/1896: Converting Node Type MatMul\n",
      "1382/1896: Converting Node Type Div\n",
      "1383/1896: Converting Node Type Add\n",
      "1384/1896: Converting Node Type Softmax\n",
      "1385/1896: Converting Node Type MatMul\n",
      "1386/1896: Converting Node Type Transpose\n",
      "1387/1896: Converting Node Type Shape\n",
      "1388/1896: Converting Node Type Gather\n",
      "1389/1896: Converting Node Type Shape\n",
      "1390/1896: Converting Node Type Gather\n",
      "1391/1896: Converting Node Type Unsqueeze\n",
      "1392/1896: Converting Node Type Unsqueeze\n",
      "1393/1896: Converting Node Type Concat\n",
      "1394/1896: Converting Node Type Reshape\n",
      "1395/1896: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1396/1896: Converting Node Type Add\n",
      "1397/1896: Converting Node Type Add\n",
      "1398/1896: Converting Node Type ReduceMean\n",
      "1399/1896: Converting Node Type Sub\n",
      "1400/1896: Converting Node Type Pow\n",
      "1401/1896: Converting Node Type ReduceMean\n",
      "1402/1896: Converting Node Type Add\n",
      "1403/1896: Converting Node Type Sqrt\n",
      "1404/1896: Converting Node Type Div\n",
      "1405/1896: Converting Node Type Mul\n",
      "1406/1896: Converting Node Type Add\n",
      "1407/1896: Converting Node Type MatMul\n",
      "1408/1896: Converting Node Type Add\n",
      "1409/1896: Converting Node Type Mul\n",
      "1410/1896: Converting Node Type Div\n",
      "1411/1896: Converting Node Type Erf\n",
      "1412/1896: Converting Node Type Add\n",
      "1413/1896: Converting Node Type Mul\n",
      "1414/1896: Converting Node Type MatMul\n",
      "1415/1896: Converting Node Type Add\n",
      "1416/1896: Converting Node Type Add\n",
      "1417/1896: Converting Node Type ReduceMean\n",
      "1418/1896: Converting Node Type Sub\n",
      "1419/1896: Converting Node Type Pow\n",
      "1420/1896: Converting Node Type ReduceMean\n",
      "1421/1896: Converting Node Type Add\n",
      "1422/1896: Converting Node Type Sqrt\n",
      "1423/1896: Converting Node Type Div\n",
      "1424/1896: Converting Node Type Mul\n",
      "1425/1896: Converting Node Type Add\n",
      "1426/1896: Converting Node Type MatMul\n",
      "1427/1896: Converting Node Type Add\n",
      "1428/1896: Converting Node Type MatMul\n",
      "1429/1896: Converting Node Type Add\n",
      "1430/1896: Converting Node Type MatMul\n",
      "1431/1896: Converting Node Type Add\n",
      "1432/1896: Converting Node Type Shape\n",
      "1433/1896: Converting Node Type Gather\n",
      "1434/1896: Converting Node Type Shape\n",
      "1435/1896: Converting Node Type Gather\n",
      "1436/1896: Converting Node Type Unsqueeze\n",
      "1437/1896: Converting Node Type Unsqueeze\n",
      "1438/1896: Converting Node Type Concat\n",
      "1439/1896: Converting Node Type Reshape\n",
      "1440/1896: Converting Node Type Transpose\n",
      "1441/1896: Converting Node Type Shape\n",
      "1442/1896: Converting Node Type Gather\n",
      "1443/1896: Converting Node Type Shape\n",
      "1444/1896: Converting Node Type Gather\n",
      "1445/1896: Converting Node Type Unsqueeze\n",
      "1446/1896: Converting Node Type Unsqueeze\n",
      "1447/1896: Converting Node Type Concat\n",
      "1448/1896: Converting Node Type Reshape\n",
      "1449/1896: Converting Node Type Shape\n",
      "1450/1896: Converting Node Type Gather\n",
      "1451/1896: Converting Node Type Shape\n",
      "1452/1896: Converting Node Type Gather\n",
      "1453/1896: Converting Node Type Unsqueeze\n",
      "1454/1896: Converting Node Type Unsqueeze\n",
      "1455/1896: Converting Node Type Concat\n",
      "1456/1896: Converting Node Type Reshape\n",
      "1457/1896: Converting Node Type Transpose\n",
      "1458/1896: Converting Node Type Transpose\n",
      "1459/1896: Converting Node Type MatMul\n",
      "1460/1896: Converting Node Type Div\n",
      "1461/1896: Converting Node Type Add\n",
      "1462/1896: Converting Node Type Softmax\n",
      "1463/1896: Converting Node Type MatMul\n",
      "1464/1896: Converting Node Type Transpose\n",
      "1465/1896: Converting Node Type Shape\n",
      "1466/1896: Converting Node Type Gather\n",
      "1467/1896: Converting Node Type Shape\n",
      "1468/1896: Converting Node Type Gather\n",
      "1469/1896: Converting Node Type Unsqueeze\n",
      "1470/1896: Converting Node Type Unsqueeze\n",
      "1471/1896: Converting Node Type Concat\n",
      "1472/1896: Converting Node Type Reshape\n",
      "1473/1896: Converting Node Type MatMul\n",
      "1474/1896: Converting Node Type Add\n",
      "1475/1896: Converting Node Type Add\n",
      "1476/1896: Converting Node Type ReduceMean\n",
      "1477/1896: Converting Node Type Sub\n",
      "1478/1896: Converting Node Type Pow\n",
      "1479/1896: Converting Node Type ReduceMean\n",
      "1480/1896: Converting Node Type Add\n",
      "1481/1896: Converting Node Type Sqrt\n",
      "1482/1896: Converting Node Type Div\n",
      "1483/1896: Converting Node Type Mul\n",
      "1484/1896: Converting Node Type Add\n",
      "1485/1896: Converting Node Type MatMul\n",
      "1486/1896: Converting Node Type Add\n",
      "1487/1896: Converting Node Type Mul\n",
      "1488/1896: Converting Node Type Div\n",
      "1489/1896: Converting Node Type Erf\n",
      "1490/1896: Converting Node Type Add\n",
      "1491/1896: Converting Node Type Mul\n",
      "1492/1896: Converting Node Type MatMul\n",
      "1493/1896: Converting Node Type Add\n",
      "1494/1896: Converting Node Type Add\n",
      "1495/1896: Converting Node Type ReduceMean\n",
      "1496/1896: Converting Node Type Sub\n",
      "1497/1896: Converting Node Type Pow\n",
      "1498/1896: Converting Node Type ReduceMean\n",
      "1499/1896: Converting Node Type Add\n",
      "1500/1896: Converting Node Type Sqrt\n",
      "1501/1896: Converting Node Type Div\n",
      "1502/1896: Converting Node Type Mul\n",
      "1503/1896: Converting Node Type Add\n",
      "1504/1896: Converting Node Type MatMul\n",
      "1505/1896: Converting Node Type Add\n",
      "1506/1896: Converting Node Type MatMul\n",
      "1507/1896: Converting Node Type Add\n",
      "1508/1896: Converting Node Type MatMul\n",
      "1509/1896: Converting Node Type Add\n",
      "1510/1896: Converting Node Type Shape\n",
      "1511/1896: Converting Node Type Gather\n",
      "1512/1896: Converting Node Type Shape\n",
      "1513/1896: Converting Node Type Gather\n",
      "1514/1896: Converting Node Type Unsqueeze\n",
      "1515/1896: Converting Node Type Unsqueeze\n",
      "1516/1896: Converting Node Type Concat\n",
      "1517/1896: Converting Node Type Reshape\n",
      "1518/1896: Converting Node Type Transpose\n",
      "1519/1896: Converting Node Type Shape\n",
      "1520/1896: Converting Node Type Gather\n",
      "1521/1896: Converting Node Type Shape\n",
      "1522/1896: Converting Node Type Gather\n",
      "1523/1896: Converting Node Type Unsqueeze\n",
      "1524/1896: Converting Node Type Unsqueeze\n",
      "1525/1896: Converting Node Type Concat\n",
      "1526/1896: Converting Node Type Reshape\n",
      "1527/1896: Converting Node Type Shape\n",
      "1528/1896: Converting Node Type Gather\n",
      "1529/1896: Converting Node Type Shape\n",
      "1530/1896: Converting Node Type Gather\n",
      "1531/1896: Converting Node Type Unsqueeze\n",
      "1532/1896: Converting Node Type Unsqueeze\n",
      "1533/1896: Converting Node Type Concat\n",
      "1534/1896: Converting Node Type Reshape\n",
      "1535/1896: Converting Node Type Transpose\n",
      "1536/1896: Converting Node Type Transpose\n",
      "1537/1896: Converting Node Type MatMul\n",
      "1538/1896: Converting Node Type Div\n",
      "1539/1896: Converting Node Type Add\n",
      "1540/1896: Converting Node Type Softmax\n",
      "1541/1896: Converting Node Type MatMul\n",
      "1542/1896: Converting Node Type Transpose\n",
      "1543/1896: Converting Node Type Shape\n",
      "1544/1896: Converting Node Type Gather\n",
      "1545/1896: Converting Node Type Shape\n",
      "1546/1896: Converting Node Type Gather\n",
      "1547/1896: Converting Node Type Unsqueeze\n",
      "1548/1896: Converting Node Type Unsqueeze\n",
      "1549/1896: Converting Node Type Concat\n",
      "1550/1896: Converting Node Type Reshape\n",
      "1551/1896: Converting Node Type MatMul\n",
      "1552/1896: Converting Node Type Add\n",
      "1553/1896: Converting Node Type Add\n",
      "1554/1896: Converting Node Type ReduceMean\n",
      "1555/1896: Converting Node Type Sub\n",
      "1556/1896: Converting Node Type Pow\n",
      "1557/1896: Converting Node Type ReduceMean\n",
      "1558/1896: Converting Node Type Add\n",
      "1559/1896: Converting Node Type Sqrt\n",
      "1560/1896: Converting Node Type Div\n",
      "1561/1896: Converting Node Type Mul\n",
      "1562/1896: Converting Node Type Add\n",
      "1563/1896: Converting Node Type MatMul\n",
      "1564/1896: Converting Node Type Add\n",
      "1565/1896: Converting Node Type Mul\n",
      "1566/1896: Converting Node Type Div\n",
      "1567/1896: Converting Node Type Erf\n",
      "1568/1896: Converting Node Type Add\n",
      "1569/1896: Converting Node Type Mul\n",
      "1570/1896: Converting Node Type MatMul\n",
      "1571/1896: Converting Node Type Add\n",
      "1572/1896: Converting Node Type Add\n",
      "1573/1896: Converting Node Type ReduceMean\n",
      "1574/1896: Converting Node Type Sub\n",
      "1575/1896: Converting Node Type Pow\n",
      "1576/1896: Converting Node Type ReduceMean\n",
      "1577/1896: Converting Node Type Add\n",
      "1578/1896: Converting Node Type Sqrt\n",
      "1579/1896: Converting Node Type Div\n",
      "1580/1896: Converting Node Type Mul\n",
      "1581/1896: Converting Node Type Add\n",
      "1582/1896: Converting Node Type MatMul\n",
      "1583/1896: Converting Node Type Add\n",
      "1584/1896: Converting Node Type MatMul\n",
      "1585/1896: Converting Node Type Add\n",
      "1586/1896: Converting Node Type MatMul\n",
      "1587/1896: Converting Node Type Add\n",
      "1588/1896: Converting Node Type Shape\n",
      "1589/1896: Converting Node Type Gather\n",
      "1590/1896: Converting Node Type Shape\n",
      "1591/1896: Converting Node Type Gather\n",
      "1592/1896: Converting Node Type Unsqueeze\n",
      "1593/1896: Converting Node Type Unsqueeze\n",
      "1594/1896: Converting Node Type Concat\n",
      "1595/1896: Converting Node Type Reshape\n",
      "1596/1896: Converting Node Type Transpose\n",
      "1597/1896: Converting Node Type Shape\n",
      "1598/1896: Converting Node Type Gather\n",
      "1599/1896: Converting Node Type Shape\n",
      "1600/1896: Converting Node Type Gather\n",
      "1601/1896: Converting Node Type Unsqueeze\n",
      "1602/1896: Converting Node Type Unsqueeze\n",
      "1603/1896: Converting Node Type Concat\n",
      "1604/1896: Converting Node Type Reshape\n",
      "1605/1896: Converting Node Type Shape\n",
      "1606/1896: Converting Node Type Gather\n",
      "1607/1896: Converting Node Type Shape\n",
      "1608/1896: Converting Node Type Gather\n",
      "1609/1896: Converting Node Type Unsqueeze\n",
      "1610/1896: Converting Node Type Unsqueeze\n",
      "1611/1896: Converting Node Type Concat\n",
      "1612/1896: Converting Node Type Reshape\n",
      "1613/1896: Converting Node Type Transpose\n",
      "1614/1896: Converting Node Type Transpose\n",
      "1615/1896: Converting Node Type MatMul\n",
      "1616/1896: Converting Node Type Div\n",
      "1617/1896: Converting Node Type Add\n",
      "1618/1896: Converting Node Type Softmax\n",
      "1619/1896: Converting Node Type MatMul\n",
      "1620/1896: Converting Node Type Transpose\n",
      "1621/1896: Converting Node Type Shape\n",
      "1622/1896: Converting Node Type Gather\n",
      "1623/1896: Converting Node Type Shape\n",
      "1624/1896: Converting Node Type Gather\n",
      "1625/1896: Converting Node Type Unsqueeze\n",
      "1626/1896: Converting Node Type Unsqueeze\n",
      "1627/1896: Converting Node Type Concat\n",
      "1628/1896: Converting Node Type Reshape\n",
      "1629/1896: Converting Node Type MatMul\n",
      "1630/1896: Converting Node Type Add\n",
      "1631/1896: Converting Node Type Add\n",
      "1632/1896: Converting Node Type ReduceMean\n",
      "1633/1896: Converting Node Type Sub\n",
      "1634/1896: Converting Node Type Pow\n",
      "1635/1896: Converting Node Type ReduceMean\n",
      "1636/1896: Converting Node Type Add\n",
      "1637/1896: Converting Node Type Sqrt\n",
      "1638/1896: Converting Node Type Div\n",
      "1639/1896: Converting Node Type Mul\n",
      "1640/1896: Converting Node Type Add\n",
      "1641/1896: Converting Node Type MatMul\n",
      "1642/1896: Converting Node Type Add\n",
      "1643/1896: Converting Node Type Mul\n",
      "1644/1896: Converting Node Type Div\n",
      "1645/1896: Converting Node Type Erf\n",
      "1646/1896: Converting Node Type Add\n",
      "1647/1896: Converting Node Type Mul\n",
      "1648/1896: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1649/1896: Converting Node Type Add\n",
      "1650/1896: Converting Node Type Add\n",
      "1651/1896: Converting Node Type ReduceMean\n",
      "1652/1896: Converting Node Type Sub\n",
      "1653/1896: Converting Node Type Pow\n",
      "1654/1896: Converting Node Type ReduceMean\n",
      "1655/1896: Converting Node Type Add\n",
      "1656/1896: Converting Node Type Sqrt\n",
      "1657/1896: Converting Node Type Div\n",
      "1658/1896: Converting Node Type Mul\n",
      "1659/1896: Converting Node Type Add\n",
      "1660/1896: Converting Node Type MatMul\n",
      "1661/1896: Converting Node Type Add\n",
      "1662/1896: Converting Node Type MatMul\n",
      "1663/1896: Converting Node Type Add\n",
      "1664/1896: Converting Node Type MatMul\n",
      "1665/1896: Converting Node Type Add\n",
      "1666/1896: Converting Node Type Shape\n",
      "1667/1896: Converting Node Type Gather\n",
      "1668/1896: Converting Node Type Shape\n",
      "1669/1896: Converting Node Type Gather\n",
      "1670/1896: Converting Node Type Unsqueeze\n",
      "1671/1896: Converting Node Type Unsqueeze\n",
      "1672/1896: Converting Node Type Concat\n",
      "1673/1896: Converting Node Type Reshape\n",
      "1674/1896: Converting Node Type Transpose\n",
      "1675/1896: Converting Node Type Shape\n",
      "1676/1896: Converting Node Type Gather\n",
      "1677/1896: Converting Node Type Shape\n",
      "1678/1896: Converting Node Type Gather\n",
      "1679/1896: Converting Node Type Unsqueeze\n",
      "1680/1896: Converting Node Type Unsqueeze\n",
      "1681/1896: Converting Node Type Concat\n",
      "1682/1896: Converting Node Type Reshape\n",
      "1683/1896: Converting Node Type Shape\n",
      "1684/1896: Converting Node Type Gather\n",
      "1685/1896: Converting Node Type Shape\n",
      "1686/1896: Converting Node Type Gather\n",
      "1687/1896: Converting Node Type Unsqueeze\n",
      "1688/1896: Converting Node Type Unsqueeze\n",
      "1689/1896: Converting Node Type Concat\n",
      "1690/1896: Converting Node Type Reshape\n",
      "1691/1896: Converting Node Type Transpose\n",
      "1692/1896: Converting Node Type Transpose\n",
      "1693/1896: Converting Node Type MatMul\n",
      "1694/1896: Converting Node Type Div\n",
      "1695/1896: Converting Node Type Add\n",
      "1696/1896: Converting Node Type Softmax\n",
      "1697/1896: Converting Node Type MatMul\n",
      "1698/1896: Converting Node Type Transpose\n",
      "1699/1896: Converting Node Type Shape\n",
      "1700/1896: Converting Node Type Gather\n",
      "1701/1896: Converting Node Type Shape\n",
      "1702/1896: Converting Node Type Gather\n",
      "1703/1896: Converting Node Type Unsqueeze\n",
      "1704/1896: Converting Node Type Unsqueeze\n",
      "1705/1896: Converting Node Type Concat\n",
      "1706/1896: Converting Node Type Reshape\n",
      "1707/1896: Converting Node Type MatMul\n",
      "1708/1896: Converting Node Type Add\n",
      "1709/1896: Converting Node Type Add\n",
      "1710/1896: Converting Node Type ReduceMean\n",
      "1711/1896: Converting Node Type Sub\n",
      "1712/1896: Converting Node Type Pow\n",
      "1713/1896: Converting Node Type ReduceMean\n",
      "1714/1896: Converting Node Type Add\n",
      "1715/1896: Converting Node Type Sqrt\n",
      "1716/1896: Converting Node Type Div\n",
      "1717/1896: Converting Node Type Mul\n",
      "1718/1896: Converting Node Type Add\n",
      "1719/1896: Converting Node Type MatMul\n",
      "1720/1896: Converting Node Type Add\n",
      "1721/1896: Converting Node Type Mul\n",
      "1722/1896: Converting Node Type Div\n",
      "1723/1896: Converting Node Type Erf\n",
      "1724/1896: Converting Node Type Add\n",
      "1725/1896: Converting Node Type Mul\n",
      "1726/1896: Converting Node Type MatMul\n",
      "1727/1896: Converting Node Type Add\n",
      "1728/1896: Converting Node Type Add\n",
      "1729/1896: Converting Node Type ReduceMean\n",
      "1730/1896: Converting Node Type Sub\n",
      "1731/1896: Converting Node Type Pow\n",
      "1732/1896: Converting Node Type ReduceMean\n",
      "1733/1896: Converting Node Type Add\n",
      "1734/1896: Converting Node Type Sqrt\n",
      "1735/1896: Converting Node Type Div\n",
      "1736/1896: Converting Node Type Mul\n",
      "1737/1896: Converting Node Type Add\n",
      "1738/1896: Converting Node Type MatMul\n",
      "1739/1896: Converting Node Type Add\n",
      "1740/1896: Converting Node Type MatMul\n",
      "1741/1896: Converting Node Type Add\n",
      "1742/1896: Converting Node Type MatMul\n",
      "1743/1896: Converting Node Type Add\n",
      "1744/1896: Converting Node Type Shape\n",
      "1745/1896: Converting Node Type Gather\n",
      "1746/1896: Converting Node Type Shape\n",
      "1747/1896: Converting Node Type Gather\n",
      "1748/1896: Converting Node Type Unsqueeze\n",
      "1749/1896: Converting Node Type Unsqueeze\n",
      "1750/1896: Converting Node Type Concat\n",
      "1751/1896: Converting Node Type Reshape\n",
      "1752/1896: Converting Node Type Transpose\n",
      "1753/1896: Converting Node Type Shape\n",
      "1754/1896: Converting Node Type Gather\n",
      "1755/1896: Converting Node Type Shape\n",
      "1756/1896: Converting Node Type Gather\n",
      "1757/1896: Converting Node Type Unsqueeze\n",
      "1758/1896: Converting Node Type Unsqueeze\n",
      "1759/1896: Converting Node Type Concat\n",
      "1760/1896: Converting Node Type Reshape\n",
      "1761/1896: Converting Node Type Shape\n",
      "1762/1896: Converting Node Type Gather\n",
      "1763/1896: Converting Node Type Shape\n",
      "1764/1896: Converting Node Type Gather\n",
      "1765/1896: Converting Node Type Unsqueeze\n",
      "1766/1896: Converting Node Type Unsqueeze\n",
      "1767/1896: Converting Node Type Concat\n",
      "1768/1896: Converting Node Type Reshape\n",
      "1769/1896: Converting Node Type Transpose\n",
      "1770/1896: Converting Node Type Transpose\n",
      "1771/1896: Converting Node Type MatMul\n",
      "1772/1896: Converting Node Type Div\n",
      "1773/1896: Converting Node Type Add\n",
      "1774/1896: Converting Node Type Softmax\n",
      "1775/1896: Converting Node Type MatMul\n",
      "1776/1896: Converting Node Type Transpose\n",
      "1777/1896: Converting Node Type Shape\n",
      "1778/1896: Converting Node Type Gather\n",
      "1779/1896: Converting Node Type Shape\n",
      "1780/1896: Converting Node Type Gather\n",
      "1781/1896: Converting Node Type Unsqueeze\n",
      "1782/1896: Converting Node Type Unsqueeze\n",
      "1783/1896: Converting Node Type Concat\n",
      "1784/1896: Converting Node Type Reshape\n",
      "1785/1896: Converting Node Type MatMul\n",
      "1786/1896: Converting Node Type Add\n",
      "1787/1896: Converting Node Type Add\n",
      "1788/1896: Converting Node Type ReduceMean\n",
      "1789/1896: Converting Node Type Sub\n",
      "1790/1896: Converting Node Type Pow\n",
      "1791/1896: Converting Node Type ReduceMean\n",
      "1792/1896: Converting Node Type Add\n",
      "1793/1896: Converting Node Type Sqrt\n",
      "1794/1896: Converting Node Type Div\n",
      "1795/1896: Converting Node Type Mul\n",
      "1796/1896: Converting Node Type Add\n",
      "1797/1896: Converting Node Type MatMul\n",
      "1798/1896: Converting Node Type Add\n",
      "1799/1896: Converting Node Type Mul\n",
      "1800/1896: Converting Node Type Div\n",
      "1801/1896: Converting Node Type Erf\n",
      "1802/1896: Converting Node Type Add\n",
      "1803/1896: Converting Node Type Mul\n",
      "1804/1896: Converting Node Type MatMul\n",
      "1805/1896: Converting Node Type Add\n",
      "1806/1896: Converting Node Type Add\n",
      "1807/1896: Converting Node Type ReduceMean\n",
      "1808/1896: Converting Node Type Sub\n",
      "1809/1896: Converting Node Type Pow\n",
      "1810/1896: Converting Node Type ReduceMean\n",
      "1811/1896: Converting Node Type Add\n",
      "1812/1896: Converting Node Type Sqrt\n",
      "1813/1896: Converting Node Type Div\n",
      "1814/1896: Converting Node Type Mul\n",
      "1815/1896: Converting Node Type Add\n",
      "1816/1896: Converting Node Type MatMul\n",
      "1817/1896: Converting Node Type Add\n",
      "1818/1896: Converting Node Type MatMul\n",
      "1819/1896: Converting Node Type Add\n",
      "1820/1896: Converting Node Type MatMul\n",
      "1821/1896: Converting Node Type Add\n",
      "1822/1896: Converting Node Type Shape\n",
      "1823/1896: Converting Node Type Gather\n",
      "1824/1896: Converting Node Type Shape\n",
      "1825/1896: Converting Node Type Gather\n",
      "1826/1896: Converting Node Type Unsqueeze\n",
      "1827/1896: Converting Node Type Unsqueeze\n",
      "1828/1896: Converting Node Type Concat\n",
      "1829/1896: Converting Node Type Reshape\n",
      "1830/1896: Converting Node Type Transpose\n",
      "1831/1896: Converting Node Type Shape\n",
      "1832/1896: Converting Node Type Gather\n",
      "1833/1896: Converting Node Type Shape\n",
      "1834/1896: Converting Node Type Gather\n",
      "1835/1896: Converting Node Type Unsqueeze\n",
      "1836/1896: Converting Node Type Unsqueeze\n",
      "1837/1896: Converting Node Type Concat\n",
      "1838/1896: Converting Node Type Reshape\n",
      "1839/1896: Converting Node Type Shape\n",
      "1840/1896: Converting Node Type Gather\n",
      "1841/1896: Converting Node Type Shape\n",
      "1842/1896: Converting Node Type Gather\n",
      "1843/1896: Converting Node Type Unsqueeze\n",
      "1844/1896: Converting Node Type Unsqueeze\n",
      "1845/1896: Converting Node Type Concat\n",
      "1846/1896: Converting Node Type Reshape\n",
      "1847/1896: Converting Node Type Transpose\n",
      "1848/1896: Converting Node Type Transpose\n",
      "1849/1896: Converting Node Type MatMul\n",
      "1850/1896: Converting Node Type Div\n",
      "1851/1896: Converting Node Type Add\n",
      "1852/1896: Converting Node Type Softmax\n",
      "1853/1896: Converting Node Type MatMul\n",
      "1854/1896: Converting Node Type Transpose\n",
      "1855/1896: Converting Node Type Shape\n",
      "1856/1896: Converting Node Type Gather\n",
      "1857/1896: Converting Node Type Shape\n",
      "1858/1896: Converting Node Type Gather\n",
      "1859/1896: Converting Node Type Unsqueeze\n",
      "1860/1896: Converting Node Type Unsqueeze\n",
      "1861/1896: Converting Node Type Concat\n",
      "1862/1896: Converting Node Type Reshape\n",
      "1863/1896: Converting Node Type MatMul\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1864/1896: Converting Node Type Add\n",
      "1865/1896: Converting Node Type Add\n",
      "1866/1896: Converting Node Type ReduceMean\n",
      "1867/1896: Converting Node Type Sub\n",
      "1868/1896: Converting Node Type Pow\n",
      "1869/1896: Converting Node Type ReduceMean\n",
      "1870/1896: Converting Node Type Add\n",
      "1871/1896: Converting Node Type Sqrt\n",
      "1872/1896: Converting Node Type Div\n",
      "1873/1896: Converting Node Type Mul\n",
      "1874/1896: Converting Node Type Add\n",
      "1875/1896: Converting Node Type MatMul\n",
      "1876/1896: Converting Node Type Add\n",
      "1877/1896: Converting Node Type Mul\n",
      "1878/1896: Converting Node Type Div\n",
      "1879/1896: Converting Node Type Erf\n",
      "1880/1896: Converting Node Type Add\n",
      "1881/1896: Converting Node Type Mul\n",
      "1882/1896: Converting Node Type MatMul\n",
      "1883/1896: Converting Node Type Add\n",
      "1884/1896: Converting Node Type Add\n",
      "1885/1896: Converting Node Type ReduceMean\n",
      "1886/1896: Converting Node Type Sub\n",
      "1887/1896: Converting Node Type Pow\n",
      "1888/1896: Converting Node Type ReduceMean\n",
      "1889/1896: Converting Node Type Add\n",
      "1890/1896: Converting Node Type Sqrt\n",
      "1891/1896: Converting Node Type Div\n",
      "1892/1896: Converting Node Type Mul\n",
      "1893/1896: Converting Node Type Add\n",
      "1894/1896: Converting Node Type Gather\n",
      "1895/1896: Converting Node Type Gemm\n",
      "1896/1896: Converting Node Type Tanh\n",
      "Translation to CoreML spec completed. Now compiling the CoreML model.\n",
      "Model Compilation done.\n"
     ]
    }
   ],
   "source": [
    "# ONNX to CoreML MLModel\n",
    "mlmodel = convert(model=onnx_model_path, target_ios=\"13\")\n",
    "mlmodel.save(mlmodel_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# PyTorch prediction\n",
    "pred_pt    = model(test_input)\n",
    "\n",
    "# MLModel prediction\n",
    "input_dict = {'input_ids': test_input.numpy().astype(np.float32)}\n",
    "pred_coreml = mlmodel.predict(input_dict, useCPUOnly=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start Scores:  SNR: 86.73614996879037 PSNR: 52.80535921101816\n",
      "End Scores:  SNR: 92.02044419648912 PSNR: 68.02353195339077\n"
     ]
    }
   ],
   "source": [
    "_compute_SNR(pred_pt[0].detach().numpy(), pred_coreml['start_scores'], 'Start Scores: ')\n",
    "_compute_SNR(pred_pt[1].detach().numpy(), pred_coreml['end_scores'], 'End Scores: ')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
